From 1533d7bd7ee608c15956ddba1722be74f7b80c15 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 17 Feb 2022 16:01:18 +0300 Subject: [PATCH 01/99] Noop thread using volatile flags and busy wait --- .../oracle/svm/core/genscavenge/GCImpl.java | 23 ++- .../oracle/svm/core/SubstrateDiagnostics.java | 7 + .../graal/snippets/CEntryPointSnippets.java | 3 + .../com/oracle/svm/core/heap/ParallelGC.java | 141 ++++++++++++++++++ .../svm/core/stack/JavaStackWalker.java | 17 +++ .../com/oracle/svm/core/thread/Safepoint.java | 33 +++- .../com/oracle/svm/core/thread/VMThreads.java | 14 ++ 7 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 88e1eefa7920..13c65d9ba716 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,8 +29,13 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.jfr.JfrTicks; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.thread.Safepoint; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -677,6 +682,9 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { + thawParallelGCThreads(); +// ImageSingletons.lookup(ParallelGC.class).signal(); + /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -768,6 +776,9 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { + thawParallelGCThreads(); + ImageSingletons.lookup(ParallelGC.class).signal(); + /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); @@ -848,7 +859,7 @@ private void blackenStackRoots() { JavaStackWalk walk = StackValue.get(JavaStackWalk.class); JavaStackWalker.initWalk(walk, sp, ip); - walkStack(walk); + walkStack(walk, WordFactory.nullPointer()); if (SubstrateOptions.MultiThreaded.getValue()) { /* @@ -866,7 +877,7 @@ private void blackenStackRoots() { continue; } if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk); + walkStack(walk, vmThread); } } } @@ -882,7 +893,7 @@ private void blackenStackRoots() { * calls to a stack frame visitor. */ @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) - private void walkStack(JavaStackWalk walk) { + private void walkStack(JavaStackWalk walk, IsolateThread thread) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; while (true) { @@ -895,7 +906,7 @@ private void walkStack(JavaStackWalk walk) { DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); if (deoptFrame == null) { if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); + throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame, thread, walk.getAnchor(), walk.getIPCodeInfo(), codeInfo); } CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); @@ -933,6 +944,10 @@ private void walkStack(JavaStackWalk walk) { } } + private void thawParallelGCThreads() { + Safepoint.Master.releaseParallelGCSafepoints(); + } + private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index d0b83fa3ace9..1c9dd8d108d4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -712,6 +712,13 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } log.string(", stack(").zhex(VMThreads.StackEnd.get(thread)).string(",").zhex(VMThreads.StackBase.get(thread)).string(")"); + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); + if (anchor.isNonNull()) { + log.newline().spaces(21) + .string("anchor=").zhex(anchor) + .string(", sp=").zhex(anchor.getLastJavaSP()) + .string(", ip=").zhex(anchor.getLastJavaIP()); + } log.newline(); printed++; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 96ebfdb87dbc..69c7e8b43e31 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -33,6 +33,7 @@ import java.util.Map; +import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -291,6 +292,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } + ImageSingletons.lookup(ParallelGC.class).startThreads(); + /* * After starting all the necessary threads, we can finally execute complex JDK code or code * that allocates a significant amount of memory. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java new file mode 100644 index 000000000000..5b6cd58962c3 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -0,0 +1,141 @@ +package com.oracle.svm.core.heap; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; +import sun.misc.Signal; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +public class ParallelGC { + + private final VMMutex mutex = new VMMutex("pargc"); + private final VMCondition cond = new VMCondition(mutex); + + private Task task = new Task(); + private volatile boolean idle = true; + + public void setTask(Pointer src, Pointer dst, UnsignedWord size) { + mutex.lock(); + task.src = src; + task.dst = dst; + task.size = size; + mutex.unlock(); + + idle = false; + cond.signal(); + Log.log().string("setTask() on ").string(Thread.currentThread().getName()).newline(); + } + + public boolean isIdle() { + return idle; + } + + public void _startThreads() { + Log trace = Log.log(); + Thread t = new Thread(() -> { + trace.string(" started thread ").string(Thread.currentThread().getName()).newline(); + while (true) { + mutex.lock(); + while (idle) { + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + cond.block(); + trace.string(" unblocked on ").string(Thread.currentThread().getName()).newline(); + } + trace.string(" copying...").newline(); +// UnmanagedMemoryUtil.copyLongsForward(task.src, task.dst, task.size); + mutex.unlock(); + idle = true; + trace.string(" done").newline(); + } + }); + t.setName("ParallelGC"); + t.setDaemon(true); + trace.string("starting thread...").newline(); + t.start(); + } + + static class Task { + Pointer src, dst; + UnsignedWord size; + } + + private volatile boolean notified; + private volatile boolean stopped; + +// public static Thread parThread; + + public void __startThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(); + } + } catch (Throwable e) { + VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); +// parThread = t; + t.start(); + } + + private void waitForNotification() { + Log trace = Log.log(); + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + while (!notified); + trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); + notified = false; + } + + public void signal() { + Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); + notified = true; + } + + public void startThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(); + } + } catch (Throwable e) { + VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); + t.start(); + } +} + +@AutomaticFeature +class ParallelGCFeature implements Feature { +// @Override +// public boolean isInConfiguration(IsInConfigurationAccess access) { +// return SubstrateOptions.UseSerialGC.getValue(); +// } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(ParallelGC.class, new ParallelGC()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index 96bfa38324f8..c9bc20c9e0b6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -245,6 +245,23 @@ public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePoi } + @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) + public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame, + IsolateThread thread, JavaFrameAnchor anchor, UntetheredCodeInfo uCodeInfo, CodeInfo codeInfo) + { + Log log = Log.log().string("Stack walk must walk only frames of known code:"); + log.string(" anchor=").hex(anchor).string(" sp=").hex(sp).string(" ip=").hex(ip).newline() + .string(" thread=").hex(thread).string(" codeInfo=").hex(codeInfo) + .string(" untethered=").hex(uCodeInfo); + + if (DeoptimizationSupport.enabled()) { + log.string(" deoptFrame=").object(deoptFrame); + } + log.newline(); + throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); + + } + public static boolean walkCurrentThread(Pointer startSP, StackFrameVisitor visitor) { return walkCurrentThread(startSP, visitor, null); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 71596b7cdb32..9b46f41c7109 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -665,7 +665,7 @@ private static boolean isMyself(IsolateThread thread) { /** Send each of the threads (except myself) a request to come to a safepoint. */ private static int requestSafepoints(String reason) { VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex while requesting a safepoint."); - final Log trace = Log.noopLog().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); + final Log trace = Log.log().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); int numOfThreads = 0; // Walk the threads list and ask each thread (except myself) to come to a safepoint. @@ -853,7 +853,7 @@ private static void waitForSafepoints(String reason) { /** Release each thread at a safepoint. */ private static void releaseSafepoints(String reason) { - final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); + final Log trace = Log.log().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set all the thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { @@ -880,6 +880,35 @@ private static void releaseSafepoints(String reason) { trace.string("]").newline(); } + /** Release each thread at a safepoint. */ + public static void releaseParallelGCSafepoints() { + final Log trace = Log.log().string("[Safepoint.Master.releasePGCSafepoints").newline(); + VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); + // Set PGC thread statuses that are at safepoint back to being in native code. + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { + if (trace.isEnabled()) { + trace.string(" PGC vmThread status: ").string(StatusSupport.getStatusString(vmThread)); + } + + restoreSafepointRequestedValue(vmThread); + + /* + * Release the thread back to native code. Most threads will transition from + * safepoint to native; but some threads will already be in native code if they + * returned from native code, found the safepoint in progress and blocked on the + * mutex putting themselves back in native code again. + */ + StatusSupport.setStatusNative(vmThread); + Statistics.incReleased(); + if (trace.isEnabled()) { + trace.string(" -> ").string(StatusSupport.getStatusString(vmThread)).newline(); + } + } + } + trace.string("]").newline(); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected IsolateThread getRequestingThread() { return requestingThread; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 9a4b14251f57..f10e1ae9aa94 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,6 +826,20 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } + public static class ParallelGCSupport { + private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); + private static final int REGULAR_THREAD = 0; + private static final int PARALLEL_GC_THREAD = 1; + + public static boolean isParallelGCThread(IsolateThread vmThread) { + return isParallelGCThreadTL.getVolatile(vmThread) == PARALLEL_GC_THREAD; + } + + public static void setParallelGCThread() { + isParallelGCThreadTL.setVolatile(PARALLEL_GC_THREAD); + } + } + public static class SafepointBehavior { /** Determines how this thread interacts with the safepoint handling. */ private static final FastThreadLocalInt safepointBehaviorTL = FastThreadLocalFactory.createInt("StatusSupport.safepointBehaviorTL"); From 6195b82dde2f2b47fbd019da0279a23340052380 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 28 Feb 2022 15:26:11 +0300 Subject: [PATCH 02/99] Noop thread with safepoints and synchronization --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +- .../graal/snippets/CEntryPointSnippets.java | 2 +- .../com/oracle/svm/core/heap/ParallelGC.java | 104 ++++-------------- .../com/oracle/svm/core/thread/Safepoint.java | 12 +- 4 files changed, 36 insertions(+), 93 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 13c65d9ba716..97df1fce19f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,7 +32,6 @@ import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.thread.Safepoint; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; @@ -682,8 +681,8 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - thawParallelGCThreads(); -// ImageSingletons.lookup(ParallelGC.class).signal(); + ParallelGC.thawWorkerThreads(); + ImageSingletons.lookup(ParallelGC.class).signal(); /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -776,7 +775,7 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - thawParallelGCThreads(); + ParallelGC.thawWorkerThreads(); ImageSingletons.lookup(ParallelGC.class).signal(); /* Visit all the Objects promoted since the snapshot, transitively. */ @@ -944,10 +943,6 @@ private void walkStack(JavaStackWalk walk, IsolateThread thread) { } } - private void thawParallelGCThreads() { - Safepoint.Master.releaseParallelGCSafepoints(); - } - private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 69c7e8b43e31..05f89811bce0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ImageSingletons.lookup(ParallelGC.class).startThreads(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 5b6cd58962c3..c2985a4e4874 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -5,73 +5,21 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.Safepoint; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; -import sun.misc.Signal; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; public class ParallelGC { private final VMMutex mutex = new VMMutex("pargc"); private final VMCondition cond = new VMCondition(mutex); - private Task task = new Task(); - private volatile boolean idle = true; - - public void setTask(Pointer src, Pointer dst, UnsignedWord size) { - mutex.lock(); - task.src = src; - task.dst = dst; - task.size = size; - mutex.unlock(); - - idle = false; - cond.signal(); - Log.log().string("setTask() on ").string(Thread.currentThread().getName()).newline(); - } - - public boolean isIdle() { - return idle; - } - - public void _startThreads() { - Log trace = Log.log(); - Thread t = new Thread(() -> { - trace.string(" started thread ").string(Thread.currentThread().getName()).newline(); - while (true) { - mutex.lock(); - while (idle) { - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - cond.block(); - trace.string(" unblocked on ").string(Thread.currentThread().getName()).newline(); - } - trace.string(" copying...").newline(); -// UnmanagedMemoryUtil.copyLongsForward(task.src, task.dst, task.size); - mutex.unlock(); - idle = true; - trace.string(" done").newline(); - } - }); - t.setName("ParallelGC"); - t.setDaemon(true); - trace.string("starting thread...").newline(); - t.start(); - } - - static class Task { - Pointer src, dst; - UnsignedWord size; - } - private volatile boolean notified; private volatile boolean stopped; -// public static Thread parThread; - - public void __startThreads() { + public void startWorkerThreads() { Log trace = Log.log(); Thread t = new Thread() { @Override @@ -83,47 +31,43 @@ public void run() { waitForNotification(); } } catch (Throwable e) { - VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + VMError.shouldNotReachHere(e.getClass().getName()); } } }; t.setName("ParallelGC-noop"); t.setDaemon(true); -// parThread = t; t.start(); } private void waitForNotification() { Log trace = Log.log(); - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - while (!notified); - trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); - notified = false; + mutex.lock(); + try { + while (!notified) { + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + cond.block(); + trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); + } + notified = false; + } finally { + mutex.unlock(); + } } public void signal() { - Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - notified = true; + mutex.lock(); + try { + Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); + notified = true; + cond.broadcast(); + } finally { + mutex.unlock(); + } } - public void startThreads() { - Log trace = Log.log(); - Thread t = new Thread() { - @Override - public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); - try { - while (!stopped) { - waitForNotification(); - } - } catch (Throwable e) { - VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); - } - } - }; - t.setName("ParallelGC-noop"); - t.setDaemon(true); - t.start(); + public static void thawWorkerThreads() { + Safepoint.Master.singleton().releaseParallelGCSafepoints(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 9b46f41c7109..75adb6e78596 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.thread; import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.RelevantForCompilationIsolates; +import static com.oracle.svm.core.thread.VMThreads.StatusSupport.STATUS_IN_SAFEPOINT; import java.util.concurrent.atomic.AtomicInteger; @@ -881,16 +882,19 @@ private static void releaseSafepoints(String reason) { } /** Release each thread at a safepoint. */ - public static void releaseParallelGCSafepoints() { - final Log trace = Log.log().string("[Safepoint.Master.releasePGCSafepoints").newline(); + public void releaseParallelGCSafepoints() { + final Log trace = Log.log().string("[Safepoint.Master.release||safepoints").newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set PGC thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { if (trace.isEnabled()) { - trace.string(" PGC vmThread status: ").string(StatusSupport.getStatusString(vmThread)); + trace.string(" || vmThread status: ").string(StatusSupport.getStatusString(vmThread)); } + if (VMThreads.StatusSupport.getStatusVolatile(vmThread) != STATUS_IN_SAFEPOINT) { + continue; + } restoreSafepointRequestedValue(vmThread); /* @@ -944,7 +948,7 @@ public static int countingVMOperation() { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; // Check if the thread is at a safepoint or in native code. switch (status) { - case StatusSupport.STATUS_IN_SAFEPOINT: + case STATUS_IN_SAFEPOINT: atSafepoint += 1; break; default: From 3dfd353912677340fec46c76622f04336a1a6f7b Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 2 Mar 2022 11:29:26 +0300 Subject: [PATCH 03/99] OldGeneration.scanGreyObjects() runs on worker thread (but logging crashes) --- .../oracle/svm/core/genscavenge/GCImpl.java | 13 +++- .../svm/core/genscavenge/OldGeneration.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 55 +++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 60 ++++--------------- 4 files changed, 78 insertions(+), 52 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 97df1fce19f1..3d2d2e897261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -682,7 +682,7 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { ParallelGC.thawWorkerThreads(); - ImageSingletons.lookup(ParallelGC.class).signal(); + parallelScanGreyObjects(false); /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -704,6 +704,17 @@ private void cheneyScanFromRoots() { } } + private void parallelScanGreyObjects(boolean isIncremental) { + Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); + try { + ParallelGC pgc = ImageSingletons.lookup(ParallelGC.class); + pgc.signal(false); + pgc.waitForNotification(false); + } finally { + scanGreyObjectsTimer.close(); + } + } + private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 8b3e042e34f1..1979a446eb32 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -106,7 +106,7 @@ void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } - boolean scanGreyObjects() { + public boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java new file mode 100644 index 000000000000..3ba09508a08f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -0,0 +1,55 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.genscavenge.HeapImpl; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.Safepoint; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature; + +public class ParallelGCImpl extends ParallelGC { + + @Override + public void startWorkerThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(true); + trace.string(" doing work on ").string(Thread.currentThread().getName()).newline(); + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + signal(true); + } + } catch (Throwable e) { + VMError.shouldNotReachHere(e.getClass().getName()); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); + t.start(); + } +} + +@AutomaticFeature +class ParallelGCFeature implements Feature { +// @Override +// public boolean isInConfiguration(IsInConfigurationAccess access) { +// return SubstrateOptions.UseSerialGC.getValue(); +// } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(ParallelGC.class, new ParallelGCImpl()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index c2985a4e4874..9834c5e62a31 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,65 +1,38 @@ package com.oracle.svm.core.heap; -import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.Safepoint; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.Feature; -public class ParallelGC { +public abstract class ParallelGC { + protected final VMMutex mutex = new VMMutex("pargc"); + protected final VMCondition cond = new VMCondition(mutex); - private final VMMutex mutex = new VMMutex("pargc"); - private final VMCondition cond = new VMCondition(mutex); + protected volatile boolean workerThreadActive; + protected volatile boolean stopped; - private volatile boolean notified; - private volatile boolean stopped; + public abstract void startWorkerThreads(); - public void startWorkerThreads() { - Log trace = Log.log(); - Thread t = new Thread() { - @Override - @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) - public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); - try { - while (!stopped) { - waitForNotification(); - } - } catch (Throwable e) { - VMError.shouldNotReachHere(e.getClass().getName()); - } - } - }; - t.setName("ParallelGC-noop"); - t.setDaemon(true); - t.start(); - } - - private void waitForNotification() { + public void waitForNotification(boolean onWorkerThread) { Log trace = Log.log(); mutex.lock(); try { - while (!notified) { + while (workerThreadActive != onWorkerThread) { trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); cond.block(); trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); } - notified = false; } finally { mutex.unlock(); } } - public void signal() { + public void signal(boolean onWorkerThread) { mutex.lock(); try { Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - notified = true; + workerThreadActive = !onWorkerThread; cond.broadcast(); } finally { mutex.unlock(); @@ -70,16 +43,3 @@ public static void thawWorkerThreads() { Safepoint.Master.singleton().releaseParallelGCSafepoints(); } } - -@AutomaticFeature -class ParallelGCFeature implements Feature { -// @Override -// public boolean isInConfiguration(IsInConfigurationAccess access) { -// return SubstrateOptions.UseSerialGC.getValue(); -// } - - @Override - public void afterRegistration(AfterRegistrationAccess access) { - ImageSingletons.add(ParallelGC.class, new ParallelGC()); - } -} From ce6b1532086e7a768d4db7d062b853f26a265ff1 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 21 Apr 2022 10:47:21 +0300 Subject: [PATCH 04/99] Worker thread synchronizing via a queue, never put on safepoint --- .../oracle/svm/core/genscavenge/GCImpl.java | 19 +------ .../svm/core/genscavenge/OldGeneration.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 14 +++-- .../genscavenge/parallel/ParallelGCImpl.java | 12 ++-- .../core/genscavenge/parallel/TaskQueue.java | 57 +++++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 5 -- .../com/oracle/svm/core/thread/Safepoint.java | 34 +---------- 7 files changed, 78 insertions(+), 65 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3d2d2e897261..2a21f0999204 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,6 +29,7 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.heap.ParallelGC; @@ -681,9 +682,6 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGC.thawWorkerThreads(); - parallelScanGreyObjects(false); - /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -704,17 +702,6 @@ private void cheneyScanFromRoots() { } } - private void parallelScanGreyObjects(boolean isIncremental) { - Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - try { - ParallelGC pgc = ImageSingletons.lookup(ParallelGC.class); - pgc.signal(false); - pgc.waitForNotification(false); - } finally { - scanGreyObjectsTimer.close(); - } - } - private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { @@ -786,8 +773,8 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGC.thawWorkerThreads(); - ImageSingletons.lookup(ParallelGC.class).signal(); + ParallelGCImpl pargc = (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + pargc.QUEUE.put(getCollectionEpoch()); /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 1979a446eb32..8b3e042e34f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -106,7 +106,7 @@ void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } - public boolean scanGreyObjects() { + boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index c7ba62e9fcc7..3bad969edce6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -202,9 +202,10 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); - } + ///assert owns mutex || pargc thread ? +// if (SubstrateOptions.MultiThreaded.getValue()) { +// VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); +// } appendAlignedHeapChunkUninterruptibly(aChunk); accounting.noteAlignedHeapChunk(); } @@ -255,9 +256,10 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); - } + ///assert owns mutex || pargc thread ? +// if (SubstrateOptions.MultiThreaded.getValue()) { +// VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); +// } appendUnalignedHeapChunkUninterruptibly(uChunk); accounting.noteUnalignedHeapChunk(uChunk); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3ba09508a08f..64a60b1dc1f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -12,9 +12,12 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.UnsignedWord; public class ParallelGCImpl extends ParallelGC { + public final TaskQueue QUEUE = new TaskQueue("pargc"); + @Override public void startWorkerThreads() { Log trace = Log.log(); @@ -25,17 +28,15 @@ public void run() { VMThreads.ParallelGCSupport.setParallelGCThread(); try { while (!stopped) { - waitForNotification(true); - trace.string(" doing work on ").string(Thread.currentThread().getName()).newline(); - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - signal(true); + UnsignedWord token = QUEUE.get(); + trace.string(" got token ").unsigned(token).string(" on ").string(Thread.currentThread().getName()).newline(); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } } }; - t.setName("ParallelGC-noop"); + t.setName("ParallelGC"); t.setDaemon(true); t.start(); } @@ -43,6 +44,7 @@ public void run() { @AutomaticFeature class ParallelGCFeature implements Feature { +/// // @Override // public boolean isInConfiguration(IsInConfigurationAccess access) { // return SubstrateOptions.UseSerialGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java new file mode 100644 index 000000000000..a0a29b376d45 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -0,0 +1,57 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +public class TaskQueue { + private static final UnsignedWord ZERO = WordFactory.zero(); + + private final VMMutex mutex; + private final VMCondition cond; + private UnsignedWord item; + + public TaskQueue(String name) { + mutex = new VMMutex(name + "-queue"); + cond = new VMCondition(mutex); + } + + public UnsignedWord get() { + try { + mutex.lock(); + while (item.equal(ZERO)) { + cond.block(); + } + return item; + } finally { + item = ZERO; + mutex.unlock(); + cond.signal(); + } + } + + public void put(UnsignedWord obj) { + try { + mutex.lock(); + while (item.notEqual(ZERO)) { + cond.block(); + } + item = obj; + } finally { + mutex.unlock(); + cond.signal(); + } + } + + public void waitUntilEmpty() { + try { + mutex.lock(); + while (item.notEqual(ZERO)) { + cond.block(); + } + } finally { + mutex.unlock(); + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 9834c5e62a31..0275910defab 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -3,7 +3,6 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.Safepoint; public abstract class ParallelGC { protected final VMMutex mutex = new VMMutex("pargc"); @@ -38,8 +37,4 @@ public void signal(boolean onWorkerThread) { mutex.unlock(); } } - - public static void thawWorkerThreads() { - Safepoint.Master.singleton().releaseParallelGCSafepoints(); - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 75adb6e78596..daa172b4b904 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -767,6 +767,8 @@ private static void waitForSafepoints(String reason) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; + } else if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { + ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; switch (status) { @@ -881,38 +883,6 @@ private static void releaseSafepoints(String reason) { trace.string("]").newline(); } - /** Release each thread at a safepoint. */ - public void releaseParallelGCSafepoints() { - final Log trace = Log.log().string("[Safepoint.Master.release||safepoints").newline(); - VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); - // Set PGC thread statuses that are at safepoint back to being in native code. - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { - if (trace.isEnabled()) { - trace.string(" || vmThread status: ").string(StatusSupport.getStatusString(vmThread)); - } - - if (VMThreads.StatusSupport.getStatusVolatile(vmThread) != STATUS_IN_SAFEPOINT) { - continue; - } - restoreSafepointRequestedValue(vmThread); - - /* - * Release the thread back to native code. Most threads will transition from - * safepoint to native; but some threads will already be in native code if they - * returned from native code, found the safepoint in progress and blocked on the - * mutex putting themselves back in native code again. - */ - StatusSupport.setStatusNative(vmThread); - Statistics.incReleased(); - if (trace.isEnabled()) { - trace.string(" -> ").string(StatusSupport.getStatusString(vmThread)).newline(); - } - } - } - trace.string("]").newline(); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected IsolateThread getRequestingThread() { return requestingThread; From 674df609d3fcc8c675c999637f5e538578f8f918 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 22 Apr 2022 12:17:54 +0300 Subject: [PATCH 05/99] Two parallel workers synchronizing via queue --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 3 +-- .../svm/core/genscavenge/YoungGeneration.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 13 ++++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2a21f0999204..bdbf529e0cf5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -773,8 +773,7 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGCImpl pargc = (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); - pargc.QUEUE.put(getCollectionEpoch()); + ParallelGCImpl.QUEUE.put(getCollectionEpoch()); /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 209fbad2731d..261420c69035 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -171,7 +171,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 64a60b1dc1f2..154dce3c6dd7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -14,12 +14,19 @@ import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.UnsignedWord; +import java.util.stream.IntStream; + public class ParallelGCImpl extends ParallelGC { - public final TaskQueue QUEUE = new TaskQueue("pargc"); + public static final int WORKERS_COUNT = 2; + public static final TaskQueue QUEUE = new TaskQueue("pargc"); @Override public void startWorkerThreads() { + IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); + } + + public void startWorkerThread(int n) { Log trace = Log.log(); Thread t = new Thread() { @Override @@ -29,14 +36,14 @@ public void run() { try { while (!stopped) { UnsignedWord token = QUEUE.get(); - trace.string(" got token ").unsigned(token).string(" on ").string(Thread.currentThread().getName()).newline(); + trace.string(" got token ").unsigned(token).string(" on PGCWorker-").unsigned(n).newline(); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } } }; - t.setName("ParallelGC"); + t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); } From ae4f77f9bca76111928d3ad1e180a1109204288c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 31 May 2022 14:10:52 +0300 Subject: [PATCH 06/99] `releaseSpaces` stage working on 2 background threads --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 +- .../svm/core/genscavenge/OldGeneration.java | 3 +- .../oracle/svm/core/genscavenge/Space.java | 35 ++++++- .../svm/core/genscavenge/YoungGeneration.java | 13 ++- .../genscavenge/parallel/ObjectQueue.java | 97 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 34 ++++--- .../core/genscavenge/parallel/TaskQueue.java | 1 + .../com/oracle/svm/core/thread/Safepoint.java | 9 +- 8 files changed, 170 insertions(+), 27 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index bdbf529e0cf5..1d6a0a09d38b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,10 +32,8 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -773,8 +771,6 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGCImpl.QUEUE.put(getCollectionEpoch()); - /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); @@ -1144,6 +1140,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + ParallelGCImpl.waitForIdle(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 8b3e042e34f1..ebf45ded2431 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -99,7 +99,8 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig } void releaseSpaces(ChunkReleaser chunkReleaser) { - getFromSpace().releaseChunks(chunkReleaser); +// getFromSpace().setInd(1000); + getFromSpace().releaseChunksParallel(chunkReleaser); } void prepareForPromotion() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3bad969edce6..005cb7d0a26b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,6 +28,8 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.annotate.AlwaysInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.locks.VMMutex; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -36,7 +38,6 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.MemoryWalker; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; @@ -45,7 +46,6 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; -import com.oracle.svm.core.thread.VMThreads; /** * A Space is a collection of HeapChunks. @@ -185,6 +185,37 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + private int ind;///rm + + private final VMMutex mutex = new VMMutex("Space.ind.mutex"); + + private void setInd(int ind) { + try { + mutex.lock(); + if (ind < 10) { + Log.log().string("## ind ").unsigned(this.ind).string(" -> ").unsigned(ind).newline(); + } + this.ind = ind; + } finally { + mutex.unlock(); + } + } + + private int getInd() { + try { + mutex.lock(); + return ind; + } finally { + mutex.unlock(); + } + } + + public void releaseChunksParallel(ChunkReleaser chunkReleaser) { +// Log.log().string(" put space ").object(this) +// .string(", chunkReleaser ").object(chunkReleaser).newline(); + ParallelGCImpl.QUEUE.put(this, chunkReleaser); + } + public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 261420c69035..4c515d4019f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -136,13 +136,22 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { } void releaseSpaces(ChunkReleaser chunkReleaser) { - getEden().releaseChunks(chunkReleaser); +// getEden().setInd(333); + getEden().releaseChunksParallel(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { - getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); +// getSurvivorFromSpaceAt(i).setInd(100+i); + getSurvivorFromSpaceAt(i).releaseChunksParallel(chunkReleaser); } } void swapSpaces() { +// try (Log trace = Log.log()) { +// trace.string(">>> eden ind=").unsigned(getEden().getInd()).newline(); +// for (int i = 0; i < maxSurvivorSpaces; i++) { +// trace.string(">>> ss").unsigned(i).string(" ind=") +// .unsigned(getSurvivorFromSpaceAt(i).getInd()).newline(); +// } +// } for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorFromSpaceAt(i).isEmpty() : "Survivor fromSpace should be empty."; assert getSurvivorFromSpaceAt(i).getChunkBytes().equal(0) : "Chunk bytes must be 0"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java new file mode 100644 index 000000000000..5b11cf0e19df --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java @@ -0,0 +1,97 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import java.util.function.BiConsumer; + +public class ObjectQueue { + private final VMMutex mutex; + private final VMCondition cond; + private final Object[] item; + private boolean empty; + private volatile int idleCount; + + public ObjectQueue(String name) { + mutex = new VMMutex(name + "-queue"); + cond = new VMCondition(mutex); + item = new Object[2]; + empty = true; + } + + private Object[] get() { +// Log.log().string(" >>> OQ.get() called").newline(); + try { + mutex.lock(); + idleCount++; + while (empty) { +// Log.log().string(" >>> OQ.get() WAIT").newline(); + cond.block(); + } +// Log.log().string(" >>> OQ.get() returns").newline(); + return item; + } finally { + empty = true; + idleCount--; + mutex.unlock(); + cond.broadcast(); + } + } + + public void put(Object value0, Object value1) { +// Log.log().string(" >>> OQ.put() called").newline(); + try { + mutex.lock(); + while (!empty) { +// Log.log().string(" >>> OQ.put() WAIT").newline(); + cond.block(); + } + item[0] = value0; + item[1] = value1; +// Log.log().string(" >>> OQ.put() returns").newline(); + } finally { + empty = false; + mutex.unlock(); + cond.broadcast(); + } + } + + public void consume(BiConsumer consumer) { + Object val0, val1; +// Log.log().string(" >>> OQ.consume() called").newline(); + try { + mutex.lock(); + idleCount++; + while (empty) { +// Log.log().string(" >>> OQ.consume() WAIT").newline(); + cond.block(); + } +// Log.log().string(" >>> OQ.consume() unblocks").newline(); + val0 = item[0]; + val1 = item[1]; + } finally { + empty = true; + idleCount--; + mutex.unlock(); + cond.broadcast(); + } + consumer.accept(val0, val1); + } + + public void waitUntilIdle(int expectedIdleCount) { + try { + mutex.lock(); +// Log.log().string(">>> OQ.wait() empty=").bool(empty) +// .string(", idle=").signed(idleCount).newline(); + while (!empty) { + cond.block(); + } + } finally { + mutex.unlock(); + } + while (idleCount < expectedIdleCount); + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 154dce3c6dd7..5ff308229c9d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,25 +1,23 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.annotate.Uninterruptible; -import com.oracle.svm.core.genscavenge.HeapImpl; +import com.oracle.svm.core.genscavenge.GCImpl; +import com.oracle.svm.core.genscavenge.Space; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.Safepoint; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.UnsignedWord; +import java.util.function.BiConsumer; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { + /// static -> ImageSingletons public static final int WORKERS_COUNT = 2; - public static final TaskQueue QUEUE = new TaskQueue("pargc"); + public static final ObjectQueue QUEUE = new ObjectQueue("pargc"); @Override public void startWorkerThreads() { @@ -27,16 +25,24 @@ public void startWorkerThreads() { } public void startWorkerThread(int n) { - Log trace = Log.log(); + final Log trace = Log.log(); + final BiConsumer releaseChunks = (s, cr) -> { + Space space = (Space) s; + GCImpl.ChunkReleaser chunkReleaser = (GCImpl.ChunkReleaser) cr; +// trace.string(" got space ").object(space) +// .string(", chunkReleaser ").object(chunkReleaser) +// .string(" on PGCWorker-").unsigned(n).newline(); + space.releaseChunks(chunkReleaser); +// space.setInd(n); + }; Thread t = new Thread() { @Override - @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); +// VMThreads.ParallelGCSupport.setParallelGCThread(); + VMThreads.SafepointBehavior.markThreadAsCrashed(); try { while (!stopped) { - UnsignedWord token = QUEUE.get(); - trace.string(" got token ").unsigned(token).string(" on PGCWorker-").unsigned(n).newline(); + QUEUE.consume(releaseChunks); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); @@ -47,6 +53,10 @@ public void run() { t.setDaemon(true); t.start(); } + + public static void waitForIdle() { + QUEUE.waitUntilIdle(WORKERS_COUNT); + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index a0a29b376d45..597a5d3276a9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,6 +5,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +///rm public class TaskQueue { private static final UnsignedWord ZERO = WordFactory.zero(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index daa172b4b904..71596b7cdb32 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.thread; import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.RelevantForCompilationIsolates; -import static com.oracle.svm.core.thread.VMThreads.StatusSupport.STATUS_IN_SAFEPOINT; import java.util.concurrent.atomic.AtomicInteger; @@ -666,7 +665,7 @@ private static boolean isMyself(IsolateThread thread) { /** Send each of the threads (except myself) a request to come to a safepoint. */ private static int requestSafepoints(String reason) { VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex while requesting a safepoint."); - final Log trace = Log.log().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); + final Log trace = Log.noopLog().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); int numOfThreads = 0; // Walk the threads list and ask each thread (except myself) to come to a safepoint. @@ -767,8 +766,6 @@ private static void waitForSafepoints(String reason) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; - } else if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { - ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; switch (status) { @@ -856,7 +853,7 @@ private static void waitForSafepoints(String reason) { /** Release each thread at a safepoint. */ private static void releaseSafepoints(String reason) { - final Log trace = Log.log().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); + final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set all the thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { @@ -918,7 +915,7 @@ public static int countingVMOperation() { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; // Check if the thread is at a safepoint or in native code. switch (status) { - case STATUS_IN_SAFEPOINT: + case StatusSupport.STATUS_IN_SAFEPOINT: atSafepoint += 1; break; default: From 7db5c4928ab7767e68b2b5223ed20f4e6ce2952e Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 2 Jun 2022 15:10:19 +0300 Subject: [PATCH 07/99] Added a custom Timer to HeapImpl --- .../src/com/oracle/svm/core/genscavenge/HeapImpl.java | 1 + .../src/com/oracle/svm/core/genscavenge/Timers.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index d9d2b1ab8b31..342b077c949e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -106,6 +106,7 @@ public final class HeapImpl extends Heap { private final RuntimeCodeInfoGCSupportImpl runtimeCodeInfoGcSupport; private final ImageHeapInfo imageHeapInfo = new ImageHeapInfo(); private final HeapAccounting accounting = new HeapAccounting(); + public final Timer timer = new Timer("promoteUnalignedHeapChunk()"); /** Head of the linked list of currently pending (ready to be enqueued) {@link Reference}s. */ private Reference refPendingList; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index bd5b1cd255fd..e00701d54eaf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -150,6 +150,7 @@ void resetAllExceptMutator() { referenceObjects.reset(); releaseSpaces.reset(); verifyAfter.reset(); + HeapImpl.getHeapImpl().timer.reset(); /* The mutator timer is *not* reset here. */ trace.string("]").newline(); } @@ -171,6 +172,7 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", blackenImageHeapRoots); logOneTimer(log, " ", blackenDirtyCardRoots); logOneTimer(log, " ", scanGreyObjects); + logOneTimer(log, " ", HeapImpl.getHeapImpl().timer); logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); logOneTimer(log, " ", releaseSpaces); From aef1e38f567be73101232348651e0339da90d845 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Jun 2022 11:02:21 +0300 Subject: [PATCH 08/99] Parallel `dirtyCardIfNecessary` --- .../oracle/svm/core/genscavenge/GCImpl.java | 58 ++++++++++++++++- .../genscavenge/GreyToBlackObjRefVisitor.java | 27 ++++---- .../svm/core/genscavenge/OldGeneration.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 6 -- .../svm/core/genscavenge/YoungGeneration.java | 4 +- .../genscavenge/parallel/ParallelGCImpl.java | 37 +++++++---- .../core/genscavenge/parallel/TaskQueue.java | 65 ++++++++++++++----- 7 files changed, 149 insertions(+), 50 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1d6a0a09d38b..9cf6a305b7fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,9 +30,12 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.TaskQueue; +import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -1039,6 +1042,7 @@ private static void prepareForPromotion(boolean isIncremental) { } private void scanGreyObjects(boolean isIncremental) { + ParallelGCImpl.setEnabled(true); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { if (isIncremental) { @@ -1049,6 +1053,8 @@ private void scanGreyObjects(boolean isIncremental) { } finally { scanGreyObjectsTimer.close(); } + ParallelGCImpl.setEnabled(false); + ParallelGCImpl.waitForIdle(); } private static void scanGreyObjectsLoop() { @@ -1096,6 +1102,57 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @AlwaysInline("GC performance") + @SuppressWarnings("static-method") + boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + /// from promObj + HeapImpl heap = HeapImpl.getHeapImpl(); + boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); + Header originalChunk = getChunk(original, isAligned); + Space originalSpace = HeapChunk.getSpace(originalChunk); + if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { + return false; + } + + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + if (result == null) { + accounting.onSurvivorOverflowed(); + } + } + if (result == null) { // complete collection, tenuring age reached, or survivor space full + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + assert result != null : "promotion failure in old generation must have been handled"; + } + + /// from visitor code + if (result != original) { + // ... update the reference to point to the copy, making the reference black. +// counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + Log.log().string("PP unmod ref").newline();/// +// counters.noteUnmodifiedReference(); + } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. +// RememberedSet.get().dirtyCardIfNecessary(holderObject, result); + + ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + return true; + } + + public void doPromoteParallel(Object original, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + /// CAREFUL: result -> original + RememberedSet.get().dirtyCardIfNecessary(holderObject, original); + } + private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1140,7 +1197,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.waitForIdle(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 95205bc7dc98..cb4c7588d820 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -102,19 +102,22 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); - if (copy != obj) { - // ... update the reference to point to the copy, making the reference black. - counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - } else { - counters.noteUnmodifiedReference(); + boolean scheduled = GCImpl.getGCImpl().promoteParallel(obj, header, objRef, innerOffset, compressed, holderObject); + if (!scheduled) { + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + if (copy != obj) { + // ... update the reference to point to the copy, making the reference black. + counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + counters.noteUnmodifiedReference(); + } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index ebf45ded2431..bb5da2af38a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -100,7 +100,7 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig void releaseSpaces(ChunkReleaser chunkReleaser) { // getFromSpace().setInd(1000); - getFromSpace().releaseChunksParallel(chunkReleaser); + getFromSpace().releaseChunks(chunkReleaser); } void prepareForPromotion() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 005cb7d0a26b..23527a6b70bc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -210,12 +210,6 @@ private int getInd() { } } - public void releaseChunksParallel(ChunkReleaser chunkReleaser) { -// Log.log().string(" put space ").object(this) -// .string(", chunkReleaser ").object(chunkReleaser).newline(); - ParallelGCImpl.QUEUE.put(this, chunkReleaser); - } - public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 4c515d4019f5..16a1a0f293fb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -137,10 +137,10 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { void releaseSpaces(ChunkReleaser chunkReleaser) { // getEden().setInd(333); - getEden().releaseChunksParallel(chunkReleaser); + getEden().releaseChunks(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { // getSurvivorFromSpaceAt(i).setInd(100+i); - getSurvivorFromSpaceAt(i).releaseChunksParallel(chunkReleaser); + getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 5ff308229c9d..050e11f44352 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -2,22 +2,27 @@ import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.Space; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; -import java.util.function.BiConsumer; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 2; - public static final ObjectQueue QUEUE = new ObjectQueue("pargc"); + public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + + public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); + }; + + private static boolean enabled; @Override public void startWorkerThreads() { @@ -26,15 +31,13 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); - final BiConsumer releaseChunks = (s, cr) -> { - Space space = (Space) s; - GCImpl.ChunkReleaser chunkReleaser = (GCImpl.ChunkReleaser) cr; -// trace.string(" got space ").object(space) -// .string(", chunkReleaser ").object(chunkReleaser) -// .string(" on PGCWorker-").unsigned(n).newline(); - space.releaseChunks(chunkReleaser); -// space.setInd(n); - }; +// final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { +// trace.string(">> promote on worker-").unsigned(n).newline();/// +// if (original == null || holderObject == null) { +// trace.string("PP orig=").object(original).string(", holder=").object(holderObject).newline(); +// } +// GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); +// }; Thread t = new Thread() { @Override public void run() { @@ -42,7 +45,7 @@ public void run() { VMThreads.SafepointBehavior.markThreadAsCrashed(); try { while (!stopped) { - QUEUE.consume(releaseChunks); + QUEUE.consume(PROMOTE_TASK); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); @@ -57,6 +60,14 @@ public void run() { public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } + + public static boolean isEnabled() { + return enabled; + } + + public static void setEnabled(boolean enabled) { + ParallelGCImpl.enabled = enabled; + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 597a5d3276a9..562b4855d065 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -2,57 +2,92 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -///rm public class TaskQueue { - private static final UnsignedWord ZERO = WordFactory.zero(); - private final VMMutex mutex; private final VMCondition cond; - private UnsignedWord item; + private Object object, holderObject; + private Pointer objRef; + private boolean compressed; + private boolean empty; + private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); + empty = true; } - public UnsignedWord get() { +// private UnsignedWord get() { +// try { +// mutex.lock(); +// while (empty) { +// cond.block(); +// } +// return word0; +// } finally { +// empty = true; +// mutex.unlock(); +// cond.signal(); +// } +// } + + public void put(Object original, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); - while (item.equal(ZERO)) { + while (!empty) { cond.block(); } - return item; + this.object = original; + this.objRef = objRef; + this.compressed = compressed; + this.holderObject = holderObject; } finally { - item = ZERO; + empty = false; mutex.unlock(); - cond.signal(); + cond.broadcast(); } } - public void put(UnsignedWord obj) { + public void consume(Consumer consumer) { + Object obj, owner; + Pointer ref; + boolean comp; try { mutex.lock(); - while (item.notEqual(ZERO)) { + idleCount++; + while (empty) { cond.block(); } - item = obj; + obj = this.object; + ref = this.objRef; + comp = this.compressed; + owner = this.holderObject; } finally { + empty = true; + idleCount--; mutex.unlock(); - cond.signal(); + cond.broadcast(); } + consumer.accept(obj, ref, comp, owner); } - public void waitUntilEmpty() { + public void waitUntilIdle(int expectedIdleCount) { try { mutex.lock(); - while (item.notEqual(ZERO)) { + while (!empty) { cond.block(); } } finally { mutex.unlock(); } + while (idleCount < expectedIdleCount);///signal? + } + + public interface Consumer { + void accept(Object original, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 54683ecc30198606927a5bbcf3e9ef44a16c13a9 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Jun 2022 12:55:16 +0300 Subject: [PATCH 09/99] Parallel objRef update --- .../oracle/svm/core/genscavenge/GCImpl.java | 24 +++++++++---------- .../genscavenge/parallel/ParallelGCImpl.java | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9cf6a305b7fe..53b5c9484c4a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,7 +30,6 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; @@ -1130,27 +1129,26 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in if (result != original) { // ... update the reference to point to the copy, making the reference black. // counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } else { Log.log().string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, result); } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. -// RememberedSet.get().dirtyCardIfNecessary(holderObject, result); - - ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } - public void doPromoteParallel(Object original, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + /// from visitor code + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + // The reference will not be updated if a whole chunk is promoted. However, we still // might have to dirty the card. - /// CAREFUL: result -> original - RememberedSet.get().dirtyCardIfNecessary(holderObject, original); + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 050e11f44352..85c4b6cd3882 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 2; + public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { From 2bfc96ec0a90ff2045646c7e023e5bf11ef0920c Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 8 Jun 2022 11:49:20 +0300 Subject: [PATCH 10/99] Passing `original` object safely via Pointer --- .../oracle/svm/core/genscavenge/GCImpl.java | 68 ++++++++++++++++--- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 8 ++- .../core/genscavenge/parallel/TaskQueue.java | 22 +++--- 4 files changed, 75 insertions(+), 25 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 53b5c9484c4a..1cbd338f1774 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -26,12 +26,16 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; @@ -1121,31 +1125,75 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in } } if (result == null) { // complete collection, tenuring age reached, or survivor space full - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); +// result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + assert originalSpace.isFromSpace(); + Space toSpace = heap.getOldGeneration().getToSpace(); +// result = toSpace.promoteAlignedObject(original, originalSpace); + assert ObjectHeaderImpl.isAlignedObject(original); + assert toSpace != originalSpace && originalSpace.isFromSpace(); +// result = toSpace.copyAlignedObject(original); + assert VMOperation.isGCInProgress(); + assert ObjectHeaderImpl.isAlignedObject(original); + + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Pointer copyMemory = toSpace.allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + result = null; + } + + /* + * This does a direct memory copy, without regard to whether the copied data contains object + * references. That's okay, because all references in the copy are visited and overwritten + * later on anyways (the card table is also updated at that point if necessary). + */ + Pointer originalMemory = Word.objectToUntrackedPointer(original); + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); + result = copyMemory.toObject(); assert result != null : "promotion failure in old generation must have been handled"; + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); + RememberedSet.get().enableRememberedSetForObject(copyChunk, result); + } + if (result != null) { + ObjectHeaderImpl.installForwardingPointer(original, result); + } +// doPromoteParallel(original, result, objRef, innerOffset, compressed, holderObject); +// ParallelGCImpl.PROMOTE_TASK.accept(result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + return true; } /// from visitor code if (result != original) { // ... update the reference to point to the copy, making the reference black. + Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); // counters.noteCopiedReferent(); - ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } else { Log.log().string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, result); } + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, result); return true; } - public void doPromoteParallel(Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Object original = originalPtr.toObject(); /// from visitor code - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - + if (copy != original) { + // ... update the reference to point to the copy, making the reference black. + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); +// counters.noteCopiedReferent(); + } else { + Log.log().string("PP unmod ref").newline();/// +// counters.noteUnmodifiedReference(); + } // The reference will not be updated if a whole chunk is promoted. However, we still // might have to dirty the card. RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 23527a6b70bc..7fccdfb502a2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -163,7 +163,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") - private Pointer allocateMemory(UnsignedWord objectSize) { + Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 85c4b6cd3882..69b346fd41b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -18,9 +18,10 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); - }; + public static final TaskQueue.Consumer PROMOTE_TASK = + (Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, copy, objRef, 0, compressed, holderObject); + }; private static boolean enabled; @@ -31,6 +32,7 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); + trace.string("PP start worker-").unsigned(n).newline(); // final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { // trace.string(">> promote on worker-").unsigned(n).newline();/// // if (original == null || holderObject == null) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 562b4855d065..e9cda952af0f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -3,14 +3,12 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object object, holderObject; - private Pointer objRef; + private Object copy, holderObject; + private Pointer original, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -35,13 +33,14 @@ public TaskQueue(String name) { // } // } - public void put(Object original, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Object copy, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.object = original; + this.original = original; + this.copy = copy; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,8 +52,8 @@ public void put(Object original, Pointer objRef, boolean compressed, Object hold } public void consume(Consumer consumer) { - Object obj, owner; - Pointer ref; + Object cp, owner; + Pointer orig, ref; boolean comp; try { mutex.lock(); @@ -62,7 +61,8 @@ public void consume(Consumer consumer) { while (empty) { cond.block(); } - obj = this.object; + orig = this.original; + cp = this.copy; ref = this.objRef; comp = this.compressed; owner = this.holderObject; @@ -72,7 +72,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(obj, ref, comp, owner); + consumer.accept(orig, cp, ref, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Object original, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 7d692484dadcdd043d9d2dcce6181659d56e534f Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 8 Jun 2022 17:05:25 +0300 Subject: [PATCH 11/99] `installFwPtr` works but not on workers --- substratevm/mx.substratevm/suite.py | 1 + .../oracle/svm/core/genscavenge/GCImpl.java | 16 +++++---- .../core/genscavenge/ObjectHeaderImpl.java | 36 ++++++++++++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 2 +- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index f0eb593c9901..342adf6795aa 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -323,6 +323,7 @@ "requiresConcealed" : { "java.base": [ "sun.nio.ch", + "jdk.internal.misc", ], "java.management": [ "sun.management", diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1cbd338f1774..ca65ecdeeab7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1156,13 +1156,8 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); RememberedSet.get().enableRememberedSetForObject(copyChunk, result); } - if (result != null) { - ObjectHeaderImpl.installForwardingPointer(original, result); - } -// doPromoteParallel(original, result, objRef, innerOffset, compressed, holderObject); -// ParallelGCImpl.PROMOTE_TASK.accept(result, objRef, compressed, holderObject); ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1183,7 +1178,14 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in } public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Log trace = Log.log(); + trace.string("PP obj=").hex(originalPtr) +// .string(", ref=").hex(objRef) + .newline(); Object original = originalPtr.toObject(); + if (copy != null) { + ObjectHeaderImpl.installForwardingPointer(original, copy); + } /// from visitor code if (copy != original) { // ... update the reference to point to the copy, making the reference black. @@ -1191,7 +1193,7 @@ public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); // counters.noteCopiedReferent(); } else { - Log.log().string("PP unmod ref").newline();/// + trace.string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); } // The reference will not be updated if a whole chunk is promoted. However, we still diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 2926cb06235f..8f9c69a9ef9c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.log.Log; +import jdk.internal.misc.Unsafe; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.CompressEncoding; @@ -325,17 +327,23 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { } } + private static final Unsafe UNSAFE = Unsafe.getUnsafe(); + /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") - static void installForwardingPointer(Object original, Object copy) { - assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + static Object installForwardingPointer(Object original, Object copy) { + Word originalPtr = Word.objectToUntrackedPointer(original); +// assert !isPointerToForwardedObject(originalPtr); + UnsignedWord oldHeader = readHeaderFromPointer(originalPtr); +// assert !isForwardedHeader(oldHeader); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + assert false: "getCompressEncoding().hasShift()"; // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); ///non mt safe } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); } @@ -343,8 +351,26 @@ static void installForwardingPointer(Object original, Object copy) { forwardHeader = Word.objectToUntrackedPointer(copy); } assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); +// writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + UnsignedWord newHeader = forwardHeader.or(FORWARDED_BIT); + boolean installed; + if (getReferenceSize() == Integer.BYTES) { +// ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); + int expected = (int) oldHeader.rawValue(); + int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); + installed = witness == expected; + } else { +// ObjectAccess.writeWord(original, getHubOffset(), newHeader); + long expected = oldHeader.rawValue(); + long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); + installed = witness == expected; + } + assert isPointerToForwardedObject(originalPtr); + if (!installed) { + Log.log().string("PP collision for obj").hex(originalPtr).newline(); + return getForwardedObject(originalPtr); + } + return copy; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 69b346fd41b5..268fc368ab8c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 4; + public static final int WORKERS_COUNT = 0; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = From f0368540cab973d326a466ab71e338f8e56737e0 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 14:47:26 +0300 Subject: [PATCH 12/99] `installForwardingPointer` works on parallel threads --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 ++++---- .../svm/core/genscavenge/ObjectHeaderImpl.java | 14 +++++++++----- .../core/genscavenge/parallel/ParallelGCImpl.java | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ca65ecdeeab7..f2360a0f641a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1157,7 +1157,7 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in RememberedSet.get().enableRememberedSetForObject(copyChunk, result); } ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1179,12 +1179,12 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); - trace.string("PP obj=").hex(originalPtr) -// .string(", ref=").hex(objRef) + trace.string("PP obj ").hex(originalPtr) +// .string(", ref ").hex(objRef) .newline(); Object original = originalPtr.toObject(); if (copy != null) { - ObjectHeaderImpl.installForwardingPointer(original, copy); + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 8f9c69a9ef9c..ac5c138b903f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -332,10 +332,14 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") static Object installForwardingPointer(Object original, Object copy) { + Log trace = Log.log(); Word originalPtr = Word.objectToUntrackedPointer(original); // assert !isPointerToForwardedObject(originalPtr); - UnsignedWord oldHeader = readHeaderFromPointer(originalPtr); -// assert !isForwardedHeader(oldHeader); + UnsignedWord curHeader = readHeaderFromPointer(originalPtr); + if (isForwardedHeader(curHeader)) { + trace.string("PP reinstall for obj ").hex(originalPtr).newline(); + return getForwardedObject(originalPtr, curHeader); + } UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { @@ -356,18 +360,18 @@ static Object installForwardingPointer(Object original, Object copy) { boolean installed; if (getReferenceSize() == Integer.BYTES) { // ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); - int expected = (int) oldHeader.rawValue(); + int expected = (int) curHeader.rawValue(); int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); installed = witness == expected; } else { // ObjectAccess.writeWord(original, getHubOffset(), newHeader); - long expected = oldHeader.rawValue(); + long expected = curHeader.rawValue(); long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); installed = witness == expected; } assert isPointerToForwardedObject(originalPtr); if (!installed) { - Log.log().string("PP collision for obj").hex(originalPtr).newline(); + trace.string("PP collision for obj ").hex(originalPtr).newline(); return getForwardedObject(originalPtr); } return copy; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 268fc368ab8c..69b346fd41b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 0; + public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = From b8dd7c8805d8678e57d41273b2921fa04791cfe3 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 15:47:53 +0300 Subject: [PATCH 13/99] `enableRememberedSetForObject` works in parallel --- .../oracle/svm/core/genscavenge/GCImpl.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f2360a0f641a..dbfc03447a4b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1150,13 +1150,7 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); result = copyMemory.toObject(); assert result != null : "promotion failure in old generation must have been handled"; - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); - RememberedSet.get().enableRememberedSetForObject(copyChunk, result); - } - ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(originalMemory, result, objRef, compressed, holderObject); // ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1182,6 +1176,16 @@ public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, trace.string("PP obj ").hex(originalPtr) // .string(", ref ").hex(objRef) .newline(); + + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + Object original = originalPtr.toObject(); if (copy != null) { copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy From ac780158ee97c1e5c811f7c796381deb186fcefe Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 17:31:44 +0300 Subject: [PATCH 14/99] Protect against promoting an object twice. Crashes frequently in FOT (:183,:256), so use `-H:-UseRememberedSet` --- .../oracle/svm/core/genscavenge/GCImpl.java | 40 +++++++++++-------- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- .../core/genscavenge/parallel/TaskQueue.java | 12 +++--- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dbfc03447a4b..7b800cc93050 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1141,16 +1141,14 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in result = null; } + Pointer originalMemory = Word.objectToUntrackedPointer(original); /* * This does a direct memory copy, without regard to whether the copied data contains object * references. That's okay, because all references in the copy are visited and overwritten * later on anyways (the card table is also updated at that point if necessary). */ - Pointer originalMemory = Word.objectToUntrackedPointer(original); UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); - result = copyMemory.toObject(); - assert result != null : "promotion failure in old generation must have been handled"; - ParallelGCImpl.QUEUE.put(originalMemory, result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(originalMemory, copyMemory, objRef, compressed, holderObject); // ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1171,22 +1169,32 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in return true; } - public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Pointer copyPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); - trace.string("PP obj ").hex(originalPtr) + Object original = originalPtr.toObject(); + Object copy; + if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { + copy = ObjectHeaderImpl.getForwardedObject(originalPtr); + } else { + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + if (size.belowOrEqual(0)) { + trace.string("PP warn obj ").hex(originalPtr) + .string(" size ").signed(size) // .string(", ref ").hex(objRef) - .newline(); + .newline(); + } + copy = copyPtr.toObject(); + assert copy != null : "promotion failure in old generation must have been handled"; - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } } - - Object original = originalPtr.toObject(); if (copy != null) { copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 69b346fd41b5..14edfae9843e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, copy, objRef, 0, compressed, holderObject); + (Pointer originalPtr, Pointer copyPtr, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, copyPtr, objRef, 0, compressed, holderObject); }; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index e9cda952af0f..51e6c28476f0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -7,8 +7,8 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object copy, holderObject; - private Pointer original, objRef; + private Object holderObject; + private Pointer original, copy, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,7 +33,7 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Object copy, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Pointer copy, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { @@ -52,8 +52,8 @@ public void put(Pointer original, Object copy, Pointer objRef, boolean compresse } public void consume(Consumer consumer) { - Object cp, owner; - Pointer orig, ref; + Object owner; + Pointer orig, cp, ref; boolean comp; try { mutex.lock(); @@ -88,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Pointer copy, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From d8b11d0f985f3dfa41edf312dbcb93dc9ea5e07b Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Jun 2022 13:38:43 +0300 Subject: [PATCH 15/99] Uninlined several methods --- .../com/oracle/svm/core/genscavenge/AlignedHeapChunk.java | 3 ++- .../com/oracle/svm/core/genscavenge/GreyObjectsWalker.java | 4 ++-- .../svm/core/genscavenge/GreyToBlackObjRefVisitor.java | 6 +++++- .../svm/core/genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../src/com/oracle/svm/core/genscavenge/HeapChunk.java | 2 +- .../svm/core/genscavenge/ReferenceObjectProcessing.java | 4 +++- .../src/com/oracle/svm/core/hub/InteriorObjRefWalker.java | 2 +- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 15a6eef96953..b010ca3b9965 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -138,7 +139,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); } - @AlwaysInline("GC performance") + @NeverInline("GC performance") static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 3788ec2572df..d67cb5c9333e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -84,7 +84,7 @@ void walkGreyObjects() { } } - @AlwaysInline("GC performance") + @NeverInline("GC performance") private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { @@ -112,7 +112,7 @@ private void walkAlignedGreyObjects() { } } - @AlwaysInline("GC performance") + @NeverInline("GC performance") private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cb4c7588d820..ec5623540c82 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,6 +24,10 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.annotate.NeverInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; +import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -64,7 +68,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override - @AlwaysInline("GC performance") + @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 88adc11c4a0b..b6a3e3fda292 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -54,7 +54,7 @@ public boolean visitObject(Object o) { } @Override - @AlwaysInline("GC performance") + @NeverInline("GC performance") public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 962423f5c1f2..7f7740d00329 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -279,7 +279,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi return walkObjectsFromInline(that, offset, visitor); } - @AlwaysInline("GC performance") + @NeverInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 3c0a614336fb..7923285d6e2a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -31,6 +31,7 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; +import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -77,7 +78,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { softReferencesAreWeak = enabled; } - @AlwaysInline("GC performance") + @NeverInline("GC performance") public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -86,6 +87,7 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @NeverInline("GC performance") private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 0d4ae77e5d2f..86c3f78f6406 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -60,7 +60,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor return walkObjectInline(obj, visitor); } - @AlwaysInline("Performance critical version") + @NeverInline("Performance critical version") public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); From 0820d8d7b8316af4b275abf2271e2acd21181657 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 23 Jun 2022 15:59:18 +0300 Subject: [PATCH 16/99] Parallel memory allocation in to-space. Simple test works on 4 threads, HyperAlloc on 1 thread --- .../oracle/svm/core/genscavenge/GCImpl.java | 61 ++++++++++--------- .../svm/core/genscavenge/HeapChunk.java | 3 + .../svm/core/genscavenge/OldGeneration.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 58 +++++++----------- .../svm/core/genscavenge/YoungGeneration.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 18 ++---- .../core/genscavenge/parallel/TaskQueue.java | 12 ++-- 7 files changed, 74 insertions(+), 86 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 7b800cc93050..d625f80dd89a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1057,7 +1057,6 @@ private void scanGreyObjects(boolean isIncremental) { scanGreyObjectsTimer.close(); } ParallelGCImpl.setEnabled(false); - ParallelGCImpl.waitForIdle(); } private static void scanGreyObjectsLoop() { @@ -1113,6 +1112,10 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); Space originalSpace = HeapChunk.getSpace(originalChunk); + if (originalSpace == null) { + Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) + .string(" in chunk ").zhex(originalChunk).newline(); + } if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { return false; } @@ -1135,21 +1138,11 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); - Pointer copyMemory = toSpace.allocateMemory(size); - if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { - result = null; - } - Pointer originalMemory = Word.objectToUntrackedPointer(original); - /* - * This does a direct memory copy, without regard to whether the copied data contains object - * references. That's okay, because all references in the copy are visited and overwritten - * later on anyways (the card table is also updated at that point if necessary). - */ - UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); - ParallelGCImpl.QUEUE.put(originalMemory, copyMemory, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + ParallelGCImpl.QUEUE.put(originalMemory, objRef, compressed, holderObject); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } return true; } @@ -1169,34 +1162,46 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in return true; } - public void doPromoteParallel(Pointer originalPtr, Pointer copyPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); Object original = originalPtr.toObject(); Object copy; if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { copy = ObjectHeaderImpl.getForwardedObject(originalPtr); } else { + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); if (size.belowOrEqual(0)) { - trace.string("PP warn obj ").hex(originalPtr) + trace.string("PP warn obj ").zhex(originalPtr) .string(" size ").signed(size) -// .string(", ref ").hex(objRef) +// .string(", ref ").zhex(objRef) .newline(); } - copy = copyPtr.toObject(); - assert copy != null : "promotion failure in old generation must have been handled"; - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + Pointer copyPtr = toSpace.allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyPtr.isNull())) { + copy = null; + } else { + /* + * This does a direct memory copy, without regard to whether the copied data contains object + * references. That's okay, because all references in the copy are visited and overwritten + * later on anyways (the card table is also updated at that point if necessary). + */ + UnmanagedMemoryUtil.copyLongsForward(originalPtr, copyPtr, size); + copy = copyPtr.toObject(); + assert copy != null : "promotion failure in old generation must have been handled"; + + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } } } if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 7f7740d00329..f987cc619df7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,6 +26,8 @@ import java.util.function.IntUnaryOperator; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -283,6 +285,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read + ParallelGCImpl.waitForIdle(); Object obj = offset.toObject(); if (!visitor.visitObjectInline(obj)) { return false; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index bb5da2af38a8..6661624b9822 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -27,6 +27,7 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; @@ -99,7 +100,6 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig } void releaseSpaces(ChunkReleaser chunkReleaser) { -// getFromSpace().setInd(1000); getFromSpace().releaseChunks(chunkReleaser); } @@ -108,10 +108,14 @@ void prepareForPromotion() { } boolean scanGreyObjects() { + Log trace = Log.log().string("").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); + ParallelGCImpl.waitForIdle(); + trace.string(">").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 7fccdfb502a2..af3582f27e96 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -27,9 +27,12 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AlwaysInline; +import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.thread.VMThreads; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -58,6 +61,7 @@ public final class Space { private final boolean isFromSpace; private final int age; private final ChunksAccounting accounting; + private final VMMutex mutex; /* Heads and tails of the HeapChunk lists. */ private AlignedHeapChunk.AlignedHeader firstAlignedHeapChunk; @@ -82,6 +86,7 @@ public final class Space { this.isFromSpace = isFromSpace; this.age = age; this.accounting = new ChunksAccounting(accounting); + this.mutex = new VMMutex(name + "-mutex"); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -162,19 +167,27 @@ public Log report(Log log, boolean traceHeapChunks) { /** * Allocate memory from an AlignedHeapChunk in this Space. */ - @AlwaysInline("GC performance") + @NeverInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { - Pointer result = WordFactory.nullPointer(); - /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); - if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { + mutex.lock(); + try { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the last chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + } + if (result.isNonNull()) { + Log.log().string(" alloc fast ").zhex(result).newline(); + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ + result = allocateInNewChunk(objectSize); + Log.log().string(" alloc slow ").zhex(result).newline(); return result; + } finally { + mutex.unlock(); } - /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(objectSize); } private Pointer allocateInNewChunk(UnsignedWord objectSize) { @@ -185,31 +198,6 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } - private int ind;///rm - - private final VMMutex mutex = new VMMutex("Space.ind.mutex"); - - private void setInd(int ind) { - try { - mutex.lock(); - if (ind < 10) { - Log.log().string("## ind ").unsigned(this.ind).string(" -> ").unsigned(ind).newline(); - } - this.ind = ind; - } finally { - mutex.unlock(); - } - } - - private int getInd() { - try { - mutex.lock(); - return ind; - } finally { - mutex.unlock(); - } - } - public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 16a1a0f293fb..a34fe38f9082 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -180,7 +180,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 14edfae9843e..de516ea01e02 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Pointer copyPtr, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, copyPtr, objRef, 0, compressed, holderObject); + (Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, objRef, 0, compressed, holderObject); }; private static boolean enabled; @@ -33,16 +33,7 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); trace.string("PP start worker-").unsigned(n).newline(); -// final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { -// trace.string(">> promote on worker-").unsigned(n).newline();/// -// if (original == null || holderObject == null) { -// trace.string("PP orig=").object(original).string(", holder=").object(holderObject).newline(); -// } -// GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); -// }; - Thread t = new Thread() { - @Override - public void run() { + Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); try { @@ -52,8 +43,7 @@ public void run() { } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } - } - }; + }); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 51e6c28476f0..96792c8058ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,7 +8,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private Object holderObject; - private Pointer original, copy, objRef; + private Pointer original, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,14 +33,13 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Pointer copy, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } this.original = original; - this.copy = copy; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,7 +52,7 @@ public void put(Pointer original, Pointer copy, Pointer objRef, boolean compress public void consume(Consumer consumer) { Object owner; - Pointer orig, cp, ref; + Pointer orig, ref; boolean comp; try { mutex.lock(); @@ -62,7 +61,6 @@ public void consume(Consumer consumer) { cond.block(); } orig = this.original; - cp = this.copy; ref = this.objRef; comp = this.compressed; owner = this.holderObject; @@ -72,7 +70,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(orig, cp, ref, comp, owner); + consumer.accept(orig, ref, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +86,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Pointer copy, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 980155b6d201e561e770bbad2ac09f4bd404e888 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 10:41:31 +0300 Subject: [PATCH 17/99] Better sync strategy in `HeapChunk.walkObjectsFromInline` --- .../svm/core/genscavenge/HeapChunk.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index f987cc619df7..e733d3cf9f70 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -284,14 +284,20 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @NeverInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; - while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read - ParallelGCImpl.waitForIdle(); - Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { - return false; + Pointer top = getTopPointer(that); + Pointer p; + do { + p = top; + while (offset.belowThan(top)) { + Object obj = offset.toObject(); + if (!visitor.visitObjectInline(obj)) { + return false; + } + offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); } - offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); - } + ParallelGCImpl.waitForIdle(); + top = getTopPointer(that); + } while (top.aboveThan(p)); return true; } From 66aa7fca1d9ecf51a5c8ac01beb0669efdf362bd Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 12:20:57 +0300 Subject: [PATCH 18/99] Simplified parallel invocation, assuming complete gc, innerOffset=0, and aligned chunk --- .../oracle/svm/core/genscavenge/GCImpl.java | 104 ++++++++---------- .../genscavenge/GreyToBlackObjRefVisitor.java | 35 +++--- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- .../core/genscavenge/parallel/TaskQueue.java | 14 ++- 4 files changed, 78 insertions(+), 79 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index d625f80dd89a..b56b212f8cd9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1104,30 +1104,55 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } - @AlwaysInline("GC performance") - @SuppressWarnings("static-method") - boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - /// from promObj - HeapImpl heap = HeapImpl.getHeapImpl(); - boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); - Header originalChunk = getChunk(original, isAligned); - Space originalSpace = HeapChunk.getSpace(originalChunk); - if (originalSpace == null) { - Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) - .string(" in chunk ").zhex(originalChunk).newline(); + public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Log trace = Log.log(); + + /// from visitor code + assert innerOffset >= 0; + assert !objRef.isNull(); +// counters.noteObjRef(); + + Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); + assert offsetP.isNonNull() || innerOffset == 0; + + Pointer originalPtr = offsetP.subtract(innerOffset); + if (originalPtr.isNull()) { +// counters.noteNullReferent(); + return; } - if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { - return false; + + if (HeapImpl.getHeapImpl().isInImageHeap(originalPtr)) { +// counters.noteNonHeapReferent(); + return; } - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - if (result == null) { - accounting.onSurvivorOverflowed(); + // This is the most expensive check as it accesses the heap fairly randomly, which results + // in a lot of cache misses. + UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalPtr); + boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); + + /// temp assertions + assert completeCollection; + assert innerOffset == 0; + assert isAligned; + + Object original = originalPtr.toObject(); + Object copy; + if (ObjectHeaderImpl.isForwardedHeader(header)) { +// counters.noteForwardedReferent(); + copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); + } else { + assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); + + /// from promObj + HeapImpl heap = HeapImpl.getHeapImpl(); + Header originalChunk = getChunk(original, isAligned); + Space originalSpace = HeapChunk.getSpace(originalChunk); + if (originalSpace == null) { + Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) + .string(" in chunk ").zhex(originalChunk).newline(); } - } - if (result == null) { // complete collection, tenuring age reached, or survivor space full + // result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); assert originalSpace.isFromSpace(); Space toSpace = heap.getOldGeneration().getToSpace(); @@ -1138,39 +1163,6 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - Pointer originalMemory = Word.objectToUntrackedPointer(original); - ParallelGCImpl.QUEUE.put(originalMemory, objRef, compressed, holderObject); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } - return true; - } - - /// from visitor code - if (result != original) { - // ... update the reference to point to the copy, making the reference black. - Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); -// counters.noteCopiedReferent(); - } else { - Log.log().string("PP unmod ref").newline();/// -// counters.noteUnmodifiedReference(); - } - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, result); - return true; - } - - public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - Log trace = Log.log(); - Object original = originalPtr.toObject(); - Object copy; - if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { - copy = ObjectHeaderImpl.getForwardedObject(originalPtr); - } else { - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); if (size.belowOrEqual(0)) { trace.string("PP warn obj ").zhex(originalPtr) @@ -1199,9 +1191,9 @@ public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffs RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } } - } - if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + if (copy != null) { + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + } } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index ec5623540c82..b8db2127656a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -70,6 +70,14 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h @Override @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + if (ParallelGCImpl.isEnabled()) { + ParallelGCImpl.QUEUE.put(objRef, innerOffset, compressed, holderObject); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } + return true; + } + assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); @@ -106,22 +114,19 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - boolean scheduled = GCImpl.getGCImpl().promoteParallel(obj, header, objRef, innerOffset, compressed, holderObject); - if (!scheduled) { - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); - if (copy != obj) { - // ... update the reference to point to the copy, making the reference black. - counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - } else { - counters.noteUnmodifiedReference(); - } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + if (copy != obj) { + // ... update the reference to point to the copy, making the reference black. + counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + counters.noteUnmodifiedReference(); } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index de516ea01e02..8feb331b55de 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, objRef, 0, compressed, holderObject); + (Pointer objRef, int innerOffset, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(objRef, innerOffset, compressed, holderObject); }; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 96792c8058ed..c1882813f53e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,7 +8,8 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private Object holderObject; - private Pointer original, objRef; + private Pointer objRef; + private int innerOffset; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,13 +34,13 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.original = original; + this.innerOffset = innerOffset; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,6 +54,7 @@ public void put(Pointer original, Pointer objRef, boolean compressed, Object hol public void consume(Consumer consumer) { Object owner; Pointer orig, ref; + int offset; boolean comp; try { mutex.lock(); @@ -60,8 +62,8 @@ public void consume(Consumer consumer) { while (empty) { cond.block(); } - orig = this.original; ref = this.objRef; + offset = this.innerOffset; comp = this.compressed; owner = this.holderObject; } finally { @@ -70,7 +72,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(orig, ref, comp, owner); + consumer.accept(ref, offset, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -86,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer objRef, int innerOffset, boolean compressed, Object holderObject); } } \ No newline at end of file From d52639380f3e8fe0db1e6c332b406b04f36a611c Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 14:57:07 +0300 Subject: [PATCH 19/99] Restored call to `Space.promoteAlignedObject()` --- .../oracle/svm/core/genscavenge/GCImpl.java | 40 +------------------ 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b56b212f8cd9..825475997fb5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1156,44 +1156,8 @@ public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compresse // result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); assert originalSpace.isFromSpace(); Space toSpace = heap.getOldGeneration().getToSpace(); -// result = toSpace.promoteAlignedObject(original, originalSpace); - assert ObjectHeaderImpl.isAlignedObject(original); - assert toSpace != originalSpace && originalSpace.isFromSpace(); -// result = toSpace.copyAlignedObject(original); - assert VMOperation.isGCInProgress(); - assert ObjectHeaderImpl.isAlignedObject(original); - - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); - if (size.belowOrEqual(0)) { - trace.string("PP warn obj ").zhex(originalPtr) - .string(" size ").signed(size) -// .string(", ref ").zhex(objRef) - .newline(); - } - - Pointer copyPtr = toSpace.allocateMemory(size); - if (probability(VERY_SLOW_PATH_PROBABILITY, copyPtr.isNull())) { - copy = null; - } else { - /* - * This does a direct memory copy, without regard to whether the copied data contains object - * references. That's okay, because all references in the copy are visited and overwritten - * later on anyways (the card table is also updated at that point if necessary). - */ - UnmanagedMemoryUtil.copyLongsForward(originalPtr, copyPtr, size); - copy = copyPtr.toObject(); - assert copy != null : "promotion failure in old generation must have been handled"; - - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); - } - } - if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); - } + copy = toSpace.promoteAlignedObject(original, originalSpace); + assert copy != null : "promotion failure in old generation must have been handled"; } /// from visitor code if (copy != original) { From 3bf93c06217bf3ce0c2fd048a38f3e7807c51953 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 15:06:26 +0300 Subject: [PATCH 20/99] Restored call to `GCImpl.promoteObject()` --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 825475997fb5..2f8034ed422f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1143,21 +1143,7 @@ public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compresse copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); } else { assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); - - /// from promObj - HeapImpl heap = HeapImpl.getHeapImpl(); - Header originalChunk = getChunk(original, isAligned); - Space originalSpace = HeapChunk.getSpace(originalChunk); - if (originalSpace == null) { - Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) - .string(" in chunk ").zhex(originalChunk).newline(); - } - -// result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - assert originalSpace.isFromSpace(); - Space toSpace = heap.getOldGeneration().getToSpace(); - copy = toSpace.promoteAlignedObject(original, originalSpace); - assert copy != null : "promotion failure in old generation must have been handled"; + copy = promoteObject(original, header); } /// from visitor code if (copy != original) { From 1b15e0103f05da6032497d5ba957f6bb7a49d547 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 15:38:27 +0300 Subject: [PATCH 21/99] Restored call to `GreyToBlackObjRefVisitor.visitObjectReferenceInline()` --- .../oracle/svm/core/genscavenge/GCImpl.java | 56 ------------------- .../genscavenge/GreyToBlackObjRefVisitor.java | 15 ++++- .../genscavenge/parallel/ParallelGCImpl.java | 6 +- .../core/genscavenge/parallel/TaskQueue.java | 19 ++++--- 4 files changed, 27 insertions(+), 69 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2f8034ed422f..ca1bf1d73a84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1104,62 +1104,6 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } - public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - Log trace = Log.log(); - - /// from visitor code - assert innerOffset >= 0; - assert !objRef.isNull(); -// counters.noteObjRef(); - - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert offsetP.isNonNull() || innerOffset == 0; - - Pointer originalPtr = offsetP.subtract(innerOffset); - if (originalPtr.isNull()) { -// counters.noteNullReferent(); - return; - } - - if (HeapImpl.getHeapImpl().isInImageHeap(originalPtr)) { -// counters.noteNonHeapReferent(); - return; - } - - // This is the most expensive check as it accesses the heap fairly randomly, which results - // in a lot of cache misses. - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalPtr); - boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); - - /// temp assertions - assert completeCollection; - assert innerOffset == 0; - assert isAligned; - - Object original = originalPtr.toObject(); - Object copy; - if (ObjectHeaderImpl.isForwardedHeader(header)) { -// counters.noteForwardedReferent(); - copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); - } else { - assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); - copy = promoteObject(original, header); - } - /// from visitor code - if (copy != original) { - // ... update the reference to point to the copy, making the reference black. - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); -// counters.noteCopiedReferent(); - } else { - trace.string("PP unmod ref").newline();/// -// counters.noteUnmodifiedReference(); - } - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); - } - private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index b8db2127656a..5c5254d01f98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -50,7 +50,7 @@ * Since this visitor is used during collection, one instance of it is constructed during native * image generation. */ -final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { +public final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { private final Counters counters; @Platforms(Platform.HOSTED_ONLY.class) @@ -71,13 +71,24 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.QUEUE.put(objRef, innerOffset, compressed, holderObject); + /// temp assertions + Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); + assert ObjectHeaderImpl.isAlignedHeader(offsetP); + assert GCImpl.getGCImpl().isCompleteCollection(); + assert innerOffset == 0; + + ParallelGCImpl.QUEUE.put(this, objRef, innerOffset, compressed, holderObject); if (ParallelGCImpl.WORKERS_COUNT <= 0) { ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } return true; + } else { + return doVisitObjectReference(objRef, innerOffset, compressed, holderObject); } + } + @NeverInline("GC performance") + public boolean doVisitObjectReference(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8feb331b55de..1844fce618f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -2,6 +2,7 @@ import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.genscavenge.GCImpl; +import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; @@ -18,10 +19,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer objRef, int innerOffset, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(objRef, innerOffset, compressed, holderObject); - }; + public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjRefVisitor::doVisitObjectReference; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index c1882813f53e..e31a30eea674 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,5 +1,6 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; @@ -7,10 +8,11 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object holderObject; + private GreyToBlackObjRefVisitor visitor; private Pointer objRef; private int innerOffset; private boolean compressed; + private Object holderObject; private boolean empty; private volatile int idleCount; @@ -34,14 +36,15 @@ public TaskQueue(String name) { // } // } - public void put(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.innerOffset = innerOffset; + this.visitor = visitor; this.objRef = objRef; + this.innerOffset = innerOffset; this.compressed = compressed; this.holderObject = holderObject; } finally { @@ -52,16 +55,18 @@ public void put(Pointer objRef, int innerOffset, boolean compressed, Object hold } public void consume(Consumer consumer) { - Object owner; - Pointer orig, ref; + GreyToBlackObjRefVisitor v; + Pointer ref; int offset; boolean comp; + Object owner; try { mutex.lock(); idleCount++; while (empty) { cond.block(); } + v = this.visitor; ref = this.objRef; offset = this.innerOffset; comp = this.compressed; @@ -72,7 +77,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(ref, offset, comp, owner); + consumer.accept(v, ref, offset, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +93,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer objRef, int innerOffset, boolean compressed, Object holderObject); + void accept(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject); } } \ No newline at end of file From 043da3831e4c51980f321603a42bdf259d862630 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Jun 2022 14:43:01 +0300 Subject: [PATCH 22/99] Extended TaskQueue to 1024 task items --- .../core/genscavenge/parallel/TaskQueue.java | 80 +++++++++++-------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index e31a30eea674..bf31fd95714f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,50 +5,59 @@ import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; +import java.util.stream.IntStream; +import java.util.stream.Stream; + public class TaskQueue { + private static final int SIZE = 1024; + + private static class TaskData { + private GreyToBlackObjRefVisitor visitor; + private Pointer objRef; + private int innerOffset; + private boolean compressed; + private Object holderObject; + } + private final VMMutex mutex; private final VMCondition cond; - private GreyToBlackObjRefVisitor visitor; - private Pointer objRef; - private int innerOffset; - private boolean compressed; - private Object holderObject; - private boolean empty; + private final TaskData[] data; + private int getIndex; + private int putIndex; private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); - empty = true; + data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); } -// private UnsignedWord get() { -// try { -// mutex.lock(); -// while (empty) { -// cond.block(); -// } -// return word0; -// } finally { -// empty = true; -// mutex.unlock(); -// cond.signal(); -// } -// } + private int next(int index) { + return (index + 1) % SIZE; + } + + private boolean canGet() { + return getIndex != putIndex; + } + + private boolean canPut() { + return next(putIndex) != getIndex; + } public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); - while (!empty) { + while (!canPut()) { cond.block(); } - this.visitor = visitor; - this.objRef = objRef; - this.innerOffset = innerOffset; - this.compressed = compressed; - this.holderObject = holderObject; + TaskData item = data[putIndex]; + item.visitor = visitor; + item.objRef = objRef; + item.innerOffset = innerOffset; + item.compressed = compressed; + item.holderObject = holderObject; } finally { - empty = false; + putIndex = next(putIndex); mutex.unlock(); cond.broadcast(); } @@ -63,16 +72,17 @@ public void consume(Consumer consumer) { try { mutex.lock(); idleCount++; - while (empty) { + while (!canGet()) { cond.block(); } - v = this.visitor; - ref = this.objRef; - offset = this.innerOffset; - comp = this.compressed; - owner = this.holderObject; + TaskData item = data[getIndex]; + v = item.visitor; + ref = item.objRef; + offset = item.innerOffset; + comp = item.compressed; + owner = item.holderObject; } finally { - empty = true; + getIndex = next(getIndex); idleCount--; mutex.unlock(); cond.broadcast(); @@ -83,7 +93,7 @@ public void consume(Consumer consumer) { public void waitUntilIdle(int expectedIdleCount) { try { mutex.lock(); - while (!empty) { + while (canGet()) { cond.block(); } } finally { From 0d936b0620c152ccfac66057e3298be16a2a21cd Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Jun 2022 15:27:22 +0300 Subject: [PATCH 23/99] Put objects on the queue rather than refs --- .../genscavenge/GreyToBlackObjRefVisitor.java | 26 +------------ .../genscavenge/GreyToBlackObjectVisitor.java | 24 ++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 6 +-- .../core/genscavenge/parallel/TaskQueue.java | 39 +++++++------------ 4 files changed, 42 insertions(+), 53 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5c5254d01f98..5707069b1776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,9 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.annotate.NeverInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -50,7 +47,7 @@ * Since this visitor is used during collection, one instance of it is constructed during native * image generation. */ -public final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { +final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { private final Counters counters; @Platforms(Platform.HOSTED_ONLY.class) @@ -68,27 +65,8 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - if (ParallelGCImpl.isEnabled()) { - /// temp assertions - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert ObjectHeaderImpl.isAlignedHeader(offsetP); - assert GCImpl.getGCImpl().isCompleteCollection(); - assert innerOffset == 0; - - ParallelGCImpl.QUEUE.put(this, objRef, innerOffset, compressed, holderObject); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } - return true; - } else { - return doVisitObjectReference(objRef, innerOffset, compressed, holderObject); - } - } - - @NeverInline("GC performance") - public boolean doVisitObjectReference(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index b6a3e3fda292..1f0db3f27681 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,6 +24,11 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ReferenceAccess; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -56,6 +61,25 @@ public boolean visitObject(Object o) { @Override @NeverInline("GC performance") public boolean visitObjectInline(Object o) { + if (ParallelGCImpl.isEnabled()) { + /// temp assertions +// Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); +// assert ObjectHeaderImpl.isAlignedHeader(offsetP); + assert GCImpl.getGCImpl().isCompleteCollection(); +// assert innerOffset == 0; + + ParallelGCImpl.QUEUE.put(this, o); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } + return true; + } else { + return doVisitObject(o); + } + } + + @NeverInline("GC performance") + public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1844fce618f3..3a4d59512212 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,15 +1,13 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; +import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.Pointer; import java.util.stream.IntStream; @@ -19,7 +17,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjRefVisitor::doVisitObjectReference; + public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjectVisitor::doVisitObject; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index bf31fd95714f..a9abe242b696 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,22 +1,18 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; +import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; -import org.graalvm.word.Pointer; +import com.oracle.svm.core.log.Log; import java.util.stream.IntStream; -import java.util.stream.Stream; public class TaskQueue { - private static final int SIZE = 1024; + private static final int SIZE = 1024; ///handle overflow private static class TaskData { - private GreyToBlackObjRefVisitor visitor; - private Pointer objRef; - private int innerOffset; - private boolean compressed; - private Object holderObject; + private GreyToBlackObjectVisitor visitor; + private Object object; } private final VMMutex mutex; @@ -44,18 +40,16 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void put(GreyToBlackObjectVisitor visitor, Object object) { try { mutex.lock(); while (!canPut()) { + Log.log().string("PP cannot put task\n"); cond.block(); } TaskData item = data[putIndex]; item.visitor = visitor; - item.objRef = objRef; - item.innerOffset = innerOffset; - item.compressed = compressed; - item.holderObject = holderObject; + item.object = object; } finally { putIndex = next(putIndex); mutex.unlock(); @@ -64,30 +58,25 @@ public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffse } public void consume(Consumer consumer) { - GreyToBlackObjRefVisitor v; - Pointer ref; - int offset; - boolean comp; - Object owner; + GreyToBlackObjectVisitor v; + Object obj; try { mutex.lock(); idleCount++; while (!canGet()) { + Log.log().string("PP cannot get task\n"); cond.block(); } TaskData item = data[getIndex]; v = item.visitor; - ref = item.objRef; - offset = item.innerOffset; - comp = item.compressed; - owner = item.holderObject; + obj = item.object; } finally { getIndex = next(getIndex); idleCount--; mutex.unlock(); cond.broadcast(); } - consumer.accept(v, ref, offset, comp, owner); + consumer.accept(v, obj); } public void waitUntilIdle(int expectedIdleCount) { @@ -103,6 +92,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject); + void accept(GreyToBlackObjectVisitor visitor, Object object); } } \ No newline at end of file From d47b6d29809d02ff3962b96cc0835b0c24eda1f2 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 11:52:52 +0300 Subject: [PATCH 24/99] Don't put G2BObjectVisitor instances on TaskQueue --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 2 +- .../svm/core/genscavenge/GreyObjectsWalker.java | 1 + .../core/genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 8 +++++++- .../svm/core/genscavenge/parallel/TaskQueue.java | 13 ++++--------- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ca1bf1d73a84..cc7ac3a37d18 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1204,7 +1204,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } - GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { + public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index d67cb5c9333e..1e8a9478081c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,6 +126,7 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; + Log.log().string("PP walk unaligned in chunk").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 1f0db3f27681..02a5be92a5c6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,7 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.QUEUE.put(this, o); + ParallelGCImpl.QUEUE.put(o); if (ParallelGCImpl.WORKERS_COUNT <= 0) { ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3a4d59512212..9b7407477644 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; @@ -17,7 +18,8 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjectVisitor::doVisitObject; + public static final TaskQueue.Consumer PROMOTE_TASK = + obj -> getVisitor().doVisitObject(obj); private static boolean enabled; @@ -45,6 +47,10 @@ public void startWorkerThread(int n) { t.start(); } + public static GreyToBlackObjectVisitor getVisitor() { + return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); + } + public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index a9abe242b696..0b4375bf7008 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,6 +1,5 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; @@ -11,8 +10,7 @@ public class TaskQueue { private static final int SIZE = 1024; ///handle overflow private static class TaskData { - private GreyToBlackObjectVisitor visitor; - private Object object; + private Object object;///no need to wrap } private final VMMutex mutex; @@ -40,7 +38,7 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(GreyToBlackObjectVisitor visitor, Object object) { + public void put(Object object) { try { mutex.lock(); while (!canPut()) { @@ -48,7 +46,6 @@ public void put(GreyToBlackObjectVisitor visitor, Object object) { cond.block(); } TaskData item = data[putIndex]; - item.visitor = visitor; item.object = object; } finally { putIndex = next(putIndex); @@ -58,7 +55,6 @@ public void put(GreyToBlackObjectVisitor visitor, Object object) { } public void consume(Consumer consumer) { - GreyToBlackObjectVisitor v; Object obj; try { mutex.lock(); @@ -68,7 +64,6 @@ public void consume(Consumer consumer) { cond.block(); } TaskData item = data[getIndex]; - v = item.visitor; obj = item.object; } finally { getIndex = next(getIndex); @@ -76,7 +71,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(v, obj); + consumer.accept(obj); } public void waitUntilIdle(int expectedIdleCount) { @@ -92,6 +87,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(GreyToBlackObjectVisitor visitor, Object object); + void accept(Object object); ///j.u.f.Consumer } } \ No newline at end of file From 239c849458c0c27aed4fa2622c96455e5a266410 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 13:05:03 +0300 Subject: [PATCH 25/99] Added max queue size statistic --- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +++ .../core/genscavenge/parallel/TaskQueue.java | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index cc7ac3a37d18..3da4b389410d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -212,6 +212,7 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); + ParallelGCImpl.QUEUE.stats.reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -436,6 +437,8 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } + verboseGCLog.newline().string(" task queue max size: ") + .unsigned(ParallelGCImpl.QUEUE.stats.getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 0b4375bf7008..76513cc8967e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -13,6 +13,8 @@ private static class TaskData { private Object object;///no need to wrap } + public final Stats stats; + private final VMMutex mutex; private final VMCondition cond; private final TaskData[] data; @@ -24,6 +26,7 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + stats = new Stats(); } private int next(int index) { @@ -49,6 +52,7 @@ public void put(Object object) { item.object = object; } finally { putIndex = next(putIndex); + stats.noteSize(putIndex, getIndex); mutex.unlock(); cond.broadcast(); } @@ -67,6 +71,7 @@ public void consume(Consumer consumer) { obj = item.object; } finally { getIndex = next(getIndex); + stats.noteSize(putIndex, getIndex); idleCount--; mutex.unlock(); cond.broadcast(); @@ -89,4 +94,26 @@ public void waitUntilIdle(int expectedIdleCount) { public interface Consumer { void accept(Object object); ///j.u.f.Consumer } + + public static class Stats { + private int maxSize; + + public void noteSize(int putIndex, int getIndex) { + int size = putIndex - getIndex; + if (size < 0) { + size += SIZE; + } + if (size > maxSize) { + maxSize = size; + } + } + + public int getMaxSize() { + return maxSize; + } + + public void reset() { + maxSize = 0; + } + } } \ No newline at end of file From e04d49a5da06e9209f378b847a9eb081cea509c7 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 16:43:37 +0300 Subject: [PATCH 26/99] Made TaskQueue private in ParallelGCImpl --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 4 ++-- .../genscavenge/GreyToBlackObjectVisitor.java | 5 +---- .../core/genscavenge/parallel/ParallelGCImpl.java | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3da4b389410d..c099dbda75cc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -212,7 +212,7 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); - ParallelGCImpl.QUEUE.stats.reset(); + ParallelGCImpl.getStats().reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -438,7 +438,7 @@ private void printGCAfter(String cause) { timers.logAfterCollection(verboseGCLog); } verboseGCLog.newline().string(" task queue max size: ") - .unsigned(ParallelGCImpl.QUEUE.stats.getMaxSize()).newline(); + .unsigned(ParallelGCImpl.getStats().getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 02a5be92a5c6..8ac694e4bebc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,10 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.QUEUE.put(o); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } + ParallelGCImpl.queue(o); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 9b7407477644..14c699191732 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -16,9 +16,9 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = + private static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); private static boolean enabled; @@ -51,6 +51,13 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + public static void queue(Object obj) { + QUEUE.put(obj); + if (WORKERS_COUNT <= 0) { // execute synchronously + QUEUE.consume(PROMOTE_TASK); + } + } + public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } @@ -62,6 +69,10 @@ public static boolean isEnabled() { public static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } + + public static TaskQueue.Stats getStats() { + return QUEUE.stats; + } } @AutomaticFeature From 6b692e2f968795c2bc1df684dbecb4851fc7ab00 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 16:50:52 +0300 Subject: [PATCH 27/99] Queue grey objects instead of scanning chunks --- .../core/genscavenge/GreyObjectsWalker.java | 2 +- .../svm/core/genscavenge/HeapChunk.java | 23 ++++++++----------- .../oracle/svm/core/genscavenge/Space.java | 4 ++++ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 1e8a9478081c..f3c221915709 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,7 +126,7 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; - Log.log().string("PP walk unaligned in chunk").zhex(uChunk).newline(); + Log.log().string("PP walk unaligned in chunk ").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index e733d3cf9f70..74cc025ff5c3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,7 +26,6 @@ import java.util.function.IntUnaryOperator; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -281,23 +280,19 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi return walkObjectsFromInline(that, offset, visitor); } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { + /// beware: this is called for both aligned and unaligned objects. Make sure both types are + /// put on the task queue. Currently only aligned ones are. Pointer offset = startOffset; Pointer top = getTopPointer(that); - Pointer p; - do { - p = top; - while (offset.belowThan(top)) { - Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { - return false; - } - offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); + while (offset.belowThan(top)) { + Object obj = offset.toObject(); + if (!visitor.visitObjectInline(obj)) { + return false; } - ParallelGCImpl.waitForIdle(); - top = getTopPointer(that); - } while (top.aboveThan(p)); + offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); + } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index af3582f27e96..f347742add55 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -397,11 +397,15 @@ private Object copyAlignedObject(Object originalObj) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } + if (ParallelGCImpl.isEnabled()) { + ParallelGCImpl.queue(copy); + } return copy; } /** Promote an AlignedHeapChunk by moving it to this space. */ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { + /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); originalSpace.extractAlignedHeapChunk(chunk); From 69bbd0dabd004acf1c60791ffd628c19b8bf31f4 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 12:25:14 +0300 Subject: [PATCH 28/99] Queue unaligned objects for parallel scan --- .../svm/core/genscavenge/GreyObjectsWalker.java | 1 - .../com/oracle/svm/core/genscavenge/HeapChunk.java | 2 -- .../src/com/oracle/svm/core/genscavenge/Space.java | 12 ++++++++++-- .../svm/core/genscavenge/UnalignedHeapChunk.java | 2 ++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index f3c221915709..d67cb5c9333e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,7 +126,6 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; - Log.log().string("PP walk unaligned in chunk ").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 74cc025ff5c3..a81495658463 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -282,8 +282,6 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { - /// beware: this is called for both aligned and unaligned objects. Make sure both types are - /// put on the task queue. Currently only aligned ones are. Pointer offset = startOffset; Pointer top = getTopPointer(that); while (offset.belowThan(top)) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index f347742add55..3d8c321cfbfc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -178,12 +179,12 @@ Pointer allocateMemory(UnsignedWord objectSize) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } if (result.isNonNull()) { - Log.log().string(" alloc fast ").zhex(result).newline(); +// Log.log().string(" alloc fast ").zhex(result).newline(); return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); - Log.log().string(" alloc slow ").zhex(result).newline(); +// Log.log().string(" alloc slow ").zhex(result).newline(); return result; } finally { mutex.unlock(); @@ -425,9 +426,16 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); + Log.log().string("UA promote chunk ").zhex(chunk).newline(); originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); + if (ParallelGCImpl.isEnabled()) { + if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { + throw VMError.shouldNotReachHere(); + } + } + if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { RememberedSet.get().enableRememberedSetForChunk(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 299309cb970b..ad12df2e7cac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -146,6 +147,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { @AlwaysInline("GC performance") public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { + Log.log().string("UA walk unaligned in chunk ").zhex(that).newline(); return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } From 23b19acc36f2b6a2fe1fa1fb18370042806786b2 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 13:50:01 +0300 Subject: [PATCH 29/99] TaskQueue fixes: `drain` and `idleCount` --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 ++- .../genscavenge/parallel/ParallelGCImpl.java | 9 ++-- .../core/genscavenge/parallel/TaskQueue.java | 43 ++++++++++++++----- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c099dbda75cc..9431f5256f49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -437,8 +437,9 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - verboseGCLog.newline().string(" task queue max size: ") - .unsigned(ParallelGCImpl.getStats().getMaxSize()).newline(); + TaskQueue.Stats stats = ParallelGCImpl.getStats(); + verboseGCLog.newline().string(" tasks: ").unsigned(stats.getCount()) + .string(", max size: ").unsigned(stats.getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 14c699191732..280a967b5503 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,13 +53,14 @@ public static GreyToBlackObjectVisitor getVisitor() { public static void queue(Object obj) { QUEUE.put(obj); - if (WORKERS_COUNT <= 0) { // execute synchronously - QUEUE.consume(PROMOTE_TASK); - } } public static void waitForIdle() { - QUEUE.waitUntilIdle(WORKERS_COUNT); + if (WORKERS_COUNT > 0) { + QUEUE.waitUntilIdle(WORKERS_COUNT); + } else { + QUEUE.drain(PROMOTE_TASK); // execute synchronously + } } public static boolean isEnabled() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 76513cc8967e..52d6db141f0d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -4,6 +4,7 @@ import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class TaskQueue { @@ -13,19 +14,20 @@ private static class TaskData { private Object object;///no need to wrap } - public final Stats stats; + final Stats stats; private final VMMutex mutex; private final VMCondition cond; private final TaskData[] data; + private final AtomicInteger idleCount; private int getIndex; private int putIndex; - private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -52,7 +54,7 @@ public void put(Object object) { item.object = object; } finally { putIndex = next(putIndex); - stats.noteSize(putIndex, getIndex); + stats.noteTask(putIndex, getIndex); mutex.unlock(); cond.broadcast(); } @@ -60,9 +62,9 @@ public void put(Object object) { public void consume(Consumer consumer) { Object obj; + idleCount.incrementAndGet(); + mutex.lock(); try { - mutex.lock(); - idleCount++; while (!canGet()) { Log.log().string("PP cannot get task\n"); cond.block(); @@ -71,15 +73,27 @@ public void consume(Consumer consumer) { obj = item.object; } finally { getIndex = next(getIndex); - stats.noteSize(putIndex, getIndex); - idleCount--; mutex.unlock(); + idleCount.decrementAndGet(); cond.broadcast(); } consumer.accept(obj); } + // Non MT safe. Only call when no workers are running + public void drain(Consumer consumer) { + idleCount.decrementAndGet(); + while (canGet()) { + TaskData item = data[getIndex]; + Object obj = item.object; + consumer.accept(obj); + getIndex = next(getIndex); + } + idleCount.incrementAndGet(); + } + public void waitUntilIdle(int expectedIdleCount) { + Log log = Log.log().string("PP waitForIdle ").unsigned(idleCount.get()).newline(); try { mutex.lock(); while (canGet()) { @@ -88,17 +102,19 @@ public void waitUntilIdle(int expectedIdleCount) { } finally { mutex.unlock(); } - while (idleCount < expectedIdleCount);///signal? + while (idleCount.get() < expectedIdleCount);///signal? + log.string("PP waitForIdle over").newline(); } public interface Consumer { void accept(Object object); ///j.u.f.Consumer } + // Non MT safe, needs locking public static class Stats { - private int maxSize; + private int count, maxSize; - public void noteSize(int putIndex, int getIndex) { + void noteTask(int putIndex, int getIndex) { int size = putIndex - getIndex; if (size < 0) { size += SIZE; @@ -106,6 +122,11 @@ public void noteSize(int putIndex, int getIndex) { if (size > maxSize) { maxSize = size; } + count++; + } + + public int getCount() { + return count; } public int getMaxSize() { @@ -113,7 +134,7 @@ public int getMaxSize() { } public void reset() { - maxSize = 0; + count = maxSize = 0; } } } \ No newline at end of file From 57e82756923c60acf869dc1c7e41382c31c39d4b Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 15:00:35 +0300 Subject: [PATCH 30/99] Made TaskQueue reference-free --- .../genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ObjectQueue.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 5 ++-- .../core/genscavenge/parallel/TaskQueue.java | 25 +++++++------------ 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 8ac694e4bebc..bda0a27f92e1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,7 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.queue(o); + ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3d8c321cfbfc..403b92961fba 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -399,7 +399,7 @@ private Object copyAlignedObject(Object originalObj) { RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(copy); + ParallelGCImpl.queue(copyMemory); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java index 5b11cf0e19df..b256b6805437 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java @@ -8,7 +8,7 @@ import java.util.function.BiConsumer; -public class ObjectQueue { +public class ObjectQueue {///rm private final VMMutex mutex; private final VMCondition cond; private final Object[] item; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 280a967b5503..9dcf15e9aee0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -9,6 +9,7 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; import java.util.stream.IntStream; @@ -51,8 +52,8 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static void queue(Object obj) { - QUEUE.put(obj); + public static void queue(Pointer ptr) { + QUEUE.put(ptr); } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 52d6db141f0d..64ce4e512c98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -3,22 +3,18 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; public class TaskQueue { private static final int SIZE = 1024; ///handle overflow - private static class TaskData { - private Object object;///no need to wrap - } - final Stats stats; private final VMMutex mutex; private final VMCondition cond; - private final TaskData[] data; + private final Pointer[] data; /// move out of heap? private final AtomicInteger idleCount; private int getIndex; private int putIndex; @@ -26,7 +22,7 @@ private static class TaskData { public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); - data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + data = new Pointer[SIZE]; idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -43,15 +39,14 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(Object object) { + public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { Log.log().string("PP cannot put task\n"); cond.block(); } - TaskData item = data[putIndex]; - item.object = object; + data[putIndex] = ptr; } finally { putIndex = next(putIndex); stats.noteTask(putIndex, getIndex); @@ -61,7 +56,7 @@ public void put(Object object) { } public void consume(Consumer consumer) { - Object obj; + Pointer ptr; idleCount.incrementAndGet(); mutex.lock(); try { @@ -69,23 +64,21 @@ public void consume(Consumer consumer) { Log.log().string("PP cannot get task\n"); cond.block(); } - TaskData item = data[getIndex]; - obj = item.object; + ptr = data[getIndex]; } finally { getIndex = next(getIndex); mutex.unlock(); idleCount.decrementAndGet(); cond.broadcast(); } - consumer.accept(obj); + consumer.accept(ptr.toObject()); } // Non MT safe. Only call when no workers are running public void drain(Consumer consumer) { idleCount.decrementAndGet(); while (canGet()) { - TaskData item = data[getIndex]; - Object obj = item.object; + Object obj = data[getIndex].toObject(); consumer.accept(obj); getIndex = next(getIndex); } From c0129b412690f9f68622b088b77a1734f16b0b02 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 4 Jul 2022 12:03:19 +0300 Subject: [PATCH 31/99] Protect `GCImpl.promoteObject` with critical section --- .../oracle/svm/core/genscavenge/GCImpl.java | 49 ++++++++++++------- .../genscavenge/GreyToBlackObjRefVisitor.java | 2 +- .../core/genscavenge/ObjectHeaderImpl.java | 4 +- .../oracle/svm/core/genscavenge/Space.java | 10 +++- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9431f5256f49..b110275a4d1d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1076,7 +1076,8 @@ private static void scanGreyObjectsLoop() { @AlwaysInline("GC performance") @SuppressWarnings("static-method") - Object promoteObject(Object original, UnsignedWord header) { + Object promoteObject(Pointer p, UnsignedWord header) { + Object original = p.toObject(); HeapImpl heap = HeapImpl.getHeapImpl(); boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); @@ -1085,27 +1086,39 @@ Object promoteObject(Object original, UnsignedWord header) { return original; } - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - if (isAligned) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + // enter critical section + originalSpace.lock(); + try { + // reread header as it might have changed + header = ObjectHeaderImpl.readHeaderFromObject(original); + if (ObjectHeaderImpl.isForwardedHeader(header)) { + return ObjectHeaderImpl.getForwardedObject(p, header); } - if (result == null) { - accounting.onSurvivorOverflowed(); + /// mutex here and Unsafe.cas in OHI.installFwPtr -- check for redundancy + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + if (isAligned) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + } + if (result == null) { + accounting.onSurvivorOverflowed(); + } } - } - if (result == null) { // complete collection, tenuring age reached, or survivor space full - if (isAligned) { - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + if (result == null) { // complete collection, tenuring age reached, or survivor space full + if (isAligned) { + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + } + assert result != null : "promotion failure in old generation must have been handled"; } - assert result != null : "promotion failure in old generation must have been handled"; - } - return result; + return result; + } finally { + originalSpace.unlock(); + } } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5707069b1776..0ff70d74fb9d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -103,7 +103,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + Object copy = GCImpl.getGCImpl().promoteObject(p, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index ac5c138b903f..6009ef596b8a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -337,7 +337,7 @@ static Object installForwardingPointer(Object original, Object copy) { // assert !isPointerToForwardedObject(originalPtr); UnsignedWord curHeader = readHeaderFromPointer(originalPtr); if (isForwardedHeader(curHeader)) { - trace.string("PP reinstall for obj ").hex(originalPtr).newline(); + trace.string("PP reinstall for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr, curHeader); } UnsignedWord forwardHeader; @@ -371,7 +371,7 @@ static Object installForwardingPointer(Object original, Object copy) { } assert isPointerToForwardedObject(originalPtr); if (!installed) { - trace.string("PP collision for obj ").hex(originalPtr).newline(); + trace.string("PP collision for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr); } return copy; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 403b92961fba..5d4025dbc4f6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -165,6 +165,14 @@ public Log report(Log log, boolean traceHeapChunks) { return log; } + void lock() { + mutex.lock(); + } + + void unlock() { + mutex.unlock(); + } + /** * Allocate memory from an AlignedHeapChunk in this Space. */ @@ -367,7 +375,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { - ObjectHeaderImpl.installForwardingPointer(original, copy); + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); } return copy; } From ff66f18375d0e9d07fd05f7cca49a52069474fa8 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 4 Jul 2022 14:52:23 +0300 Subject: [PATCH 32/99] Fixed `TaskQueue.waitUntilIdle` --- .../core/genscavenge/parallel/TaskQueue.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 64ce4e512c98..1304b32e9827 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,8 +5,6 @@ import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; -import java.util.concurrent.atomic.AtomicInteger; - public class TaskQueue { private static final int SIZE = 1024; ///handle overflow @@ -15,7 +13,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private final Pointer[] data; /// move out of heap? - private final AtomicInteger idleCount; + private int idleCount; private int getIndex; private int putIndex; @@ -23,7 +21,6 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = new Pointer[SIZE]; - idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -57,18 +54,18 @@ public void put(Pointer ptr) { public void consume(Consumer consumer) { Pointer ptr; - idleCount.incrementAndGet(); mutex.lock(); try { while (!canGet()) { Log.log().string("PP cannot get task\n"); + idleCount++; cond.block(); + idleCount--; } ptr = data[getIndex]; } finally { getIndex = next(getIndex); mutex.unlock(); - idleCount.decrementAndGet(); cond.broadcast(); } consumer.accept(ptr.toObject()); @@ -76,27 +73,29 @@ public void consume(Consumer consumer) { // Non MT safe. Only call when no workers are running public void drain(Consumer consumer) { - idleCount.decrementAndGet(); while (canGet()) { Object obj = data[getIndex].toObject(); consumer.accept(obj); getIndex = next(getIndex); } - idleCount.incrementAndGet(); } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("PP waitForIdle ").unsigned(idleCount.get()).newline(); - try { - mutex.lock(); - while (canGet()) { - cond.block(); + Log log = Log.log().string("PP waitForIdle\n"); + while (true) { + try { + mutex.lock(); + while (canGet()) { + cond.block(); + } + if (idleCount >= expectedIdleCount) { + log.string("PP waitForIdle over\n"); + return; + } + } finally { + mutex.unlock(); } - } finally { - mutex.unlock(); } - while (idleCount.get() < expectedIdleCount);///signal? - log.string("PP waitForIdle over").newline(); } public interface Consumer { From af6f70b53a153fa48dd7c5bbe749966dd17068e0 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 5 Jul 2022 15:42:36 +0300 Subject: [PATCH 33/99] Removed locking on fromSpace --- .../oracle/svm/core/genscavenge/GCImpl.java | 49 ++++++--------- .../genscavenge/GreyToBlackObjRefVisitor.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 62 ++++++++++++++++--- .../svm/core/hub/InteriorObjRefWalker.java | 4 ++ 4 files changed, 76 insertions(+), 41 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b110275a4d1d..9431f5256f49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1076,8 +1076,7 @@ private static void scanGreyObjectsLoop() { @AlwaysInline("GC performance") @SuppressWarnings("static-method") - Object promoteObject(Pointer p, UnsignedWord header) { - Object original = p.toObject(); + Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); @@ -1086,39 +1085,27 @@ Object promoteObject(Pointer p, UnsignedWord header) { return original; } - // enter critical section - originalSpace.lock(); - try { - // reread header as it might have changed - header = ObjectHeaderImpl.readHeaderFromObject(original); - if (ObjectHeaderImpl.isForwardedHeader(header)) { - return ObjectHeaderImpl.getForwardedObject(p, header); + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + if (isAligned) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); } - /// mutex here and Unsafe.cas in OHI.installFwPtr -- check for redundancy - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - if (isAligned) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); - } - if (result == null) { - accounting.onSurvivorOverflowed(); - } + if (result == null) { + accounting.onSurvivorOverflowed(); } - if (result == null) { // complete collection, tenuring age reached, or survivor space full - if (isAligned) { - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); - } - assert result != null : "promotion failure in old generation must have been handled"; + } + if (result == null) { // complete collection, tenuring age reached, or survivor space full + if (isAligned) { + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); } - - return result; - } finally { - originalSpace.unlock(); + assert result != null : "promotion failure in old generation must have been handled"; } + + return result; } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 0ff70d74fb9d..5707069b1776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -103,7 +103,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(p, header); + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5d4025dbc4f6..ca20abfd4f97 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -165,14 +165,6 @@ public Log report(Log log, boolean traceHeapChunks) { return log; } - void lock() { - mutex.lock(); - } - - void unlock() { - mutex.unlock(); - } - /** * Allocate memory from an AlignedHeapChunk in this Space. */ @@ -373,13 +365,65 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); + if (ParallelGCImpl.isEnabled()) { + return promoteAlignedObjectParallel(original, originalSpace); + } + Object copy = copyAlignedObject(original); if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + ObjectHeaderImpl.installForwardingPointer(original, copy); } return copy; } + @AlwaysInline("GC performance") + Object promoteAlignedObjectParallel(Object original, Space originalSpace) { + assert VMOperation.isGCInProgress(); + assert ObjectHeaderImpl.isAlignedObject(original); + + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Pointer copyMemory = allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + return null; + } + Object copy = copyMemory.toObject(); + if (copy == null) { + return null; + } + + // Read original header + Pointer originalMemory = Word.objectToUntrackedPointer(original); + assert ObjectHeaderImpl.getHubOffset() == 0; + UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + // Install forwarding pointer into the original header + /// mv CAS code from OHI here + Object forward = ObjectHeaderImpl.installForwardingPointer(original, copy); + + if (forward == copy) { + // We have won the race, now we must copy the object bits. First install the original header + int headerSize = ObjectHeaderImpl.getReferenceSize(); + if (headerSize == Integer.BYTES) { + copyMemory.writeInt(0, (int) header.rawValue()); + } else { + copyMemory.writeWord(0, header); + } + // copy the rest of original object + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); + + if (isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + ParallelGCImpl.queue(copyMemory); + return copy; + } else { + /// Now the allocated memory is lost. Retract in TLAB? + return forward; + } + } + @AlwaysInline("GC performance") private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 86c3f78f6406..87e13af29b94 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.hub; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.word.Word; import org.graalvm.word.Pointer; @@ -65,6 +66,9 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); + if (objHub.getName() == null) { + Log.log().string("PP warn NULL obj ").zhex(objPointer).newline(); + } // Visit each Object reference in the array part of the Object. if (LayoutEncoding.isArrayLikeWithObjectElements(layoutEncoding)) { From 972002c8b084611572cf12f987a40f6382c2fbb1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 6 Jul 2022 10:04:09 +0300 Subject: [PATCH 34/99] `scanGreyObjects` cleanup --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 11 ++++++----- .../core/genscavenge/GreyToBlackObjectVisitor.java | 6 ------ .../oracle/svm/core/genscavenge/OldGeneration.java | 7 +++---- .../src/com/oracle/svm/core/genscavenge/Space.java | 3 --- .../oracle/svm/core/genscavenge/YoungGeneration.java | 6 ++++-- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 2 +- .../svm/core/genscavenge/parallel/TaskQueue.java | 8 ++++---- .../com/oracle/svm/core/hub/InteriorObjRefWalker.java | 2 +- 8 files changed, 19 insertions(+), 26 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9431f5256f49..5748b7c636f6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1049,18 +1049,19 @@ private static void prepareForPromotion(boolean isIncremental) { } private void scanGreyObjects(boolean isIncremental) { - ParallelGCImpl.setEnabled(true); + HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); + ParallelGCImpl.setEnabled(true); try { if (isIncremental) { - scanGreyObjectsLoop(); - } else { - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + heap.getYoungGeneration().scanGreyObjects(); } + heap.getOldGeneration().scanGreyObjects(); } finally { + ParallelGCImpl.waitForIdle(); + ParallelGCImpl.setEnabled(false); scanGreyObjectsTimer.close(); } - ParallelGCImpl.setEnabled(false); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index bda0a27f92e1..4ec0d030e795 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -62,12 +62,6 @@ public boolean visitObject(Object o) { @NeverInline("GC performance") public boolean visitObjectInline(Object o) { if (ParallelGCImpl.isEnabled()) { - /// temp assertions -// Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); -// assert ObjectHeaderImpl.isAlignedHeader(offsetP); - assert GCImpl.getGCImpl().isCompleteCollection(); -// assert innerOffset == 0; - ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); return true; } else { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 6661624b9822..f6f2c9d9f24a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -108,14 +108,13 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("").newline(); + trace.string("]").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); - ParallelGCImpl.waitForIdle(); - trace.string(">").newline(); + trace.string("]").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ca20abfd4f97..0c662cf0cb6f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -450,9 +450,6 @@ private Object copyAlignedObject(Object originalObj) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(copyMemory); - } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index a34fe38f9082..e1215f39c62c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -180,7 +180,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:").newline(); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -189,11 +189,13 @@ boolean scanGreyObjects() { } } if (!needScan) { + trace.string("]").newline(); return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).string("]").newline(); + trace.string("[Scanning survivor-").signed(i).newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); + trace.string("]").newline(); } trace.string("]").newline(); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 9dcf15e9aee0..f9634ff9bb9e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -18,7 +18,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - private static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + private static final TaskQueue QUEUE = new TaskQueue("pargc"); private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 1304b32e9827..2038c4c03351 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -40,7 +40,7 @@ public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { - Log.log().string("PP cannot put task\n"); + Log.log().string("TQ cannot put task\n"); cond.block(); } data[putIndex] = ptr; @@ -57,7 +57,7 @@ public void consume(Consumer consumer) { mutex.lock(); try { while (!canGet()) { - Log.log().string("PP cannot get task\n"); + Log.log().string("TQ cannot get task\n"); idleCount++; cond.block(); idleCount--; @@ -81,7 +81,7 @@ public void drain(Consumer consumer) { } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("PP waitForIdle\n"); + Log log = Log.log().string("TQ waitForIdle\n"); while (true) { try { mutex.lock(); @@ -89,7 +89,7 @@ public void waitUntilIdle(int expectedIdleCount) { cond.block(); } if (idleCount >= expectedIdleCount) { - log.string("PP waitForIdle over\n"); + log.string("TQ waitForIdle over\n"); return; } } finally { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 87e13af29b94..bc5dde38d981 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -67,7 +67,7 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); if (objHub.getName() == null) { - Log.log().string("PP warn NULL obj ").zhex(objPointer).newline(); + Log.log().string("PP warn hub.name=null for obj ").zhex(objPointer).newline(); } // Visit each Object reference in the array part of the Object. From b6c5168b6c91d6ab7b0d9baaffb9bd3fadadb0de Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 8 Jul 2022 12:32:27 +0300 Subject: [PATCH 35/99] Added `ParallelGCImpl.checkThrowable` --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 5 +++++ .../core/genscavenge/parallel/ParallelGCImpl.java | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 5748b7c636f6..8fff237c339c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -171,6 +171,7 @@ private void collect(GCCause cause, boolean forceFullGC) { if (outOfMemory) { throw OutOfMemoryUtil.heapSizeExceeded(); } + ParallelGCImpl.checkThrowable(); } } @@ -375,6 +376,10 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); + boolean compressed = ReferenceAccess.singleton().haveCompressedReferences(); + boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); + verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); + verboseGCLog.string("]").newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index f9634ff9bb9e..0374e9248a73 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,7 +6,6 @@ import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -22,7 +21,8 @@ public class ParallelGCImpl extends ParallelGC { private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); - private static boolean enabled; + private static volatile boolean enabled; + private static volatile Throwable throwable; @Override public void startWorkerThreads() { @@ -40,7 +40,7 @@ public void startWorkerThread(int n) { QUEUE.consume(PROMOTE_TASK); } } catch (Throwable e) { - VMError.shouldNotReachHere(e.getClass().getName()); + throwable = e; } }); t.setName("ParallelGCWorker-" + n); @@ -75,6 +75,15 @@ public static void setEnabled(boolean enabled) { public static TaskQueue.Stats getStats() { return QUEUE.stats; } + + public static void checkThrowable() { + if (throwable != null) { + Log.log().string("PGC error : ").string(throwable.getClass().getName()) + .string(" : ").string(throwable.getMessage()).newline(); + throwable.printStackTrace(); + throw new Error(throwable); + } + } } @AutomaticFeature From 2295b159fa32b2c03fa2c0c8f10074f5098ec99e Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 14:41:29 +0300 Subject: [PATCH 36/99] Increased queue size for HyperAlloc --- .../src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 2038c4c03351..519c784767bd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -6,7 +6,7 @@ import org.graalvm.word.Pointer; public class TaskQueue { - private static final int SIZE = 1024; ///handle overflow + private static final int SIZE = 1024 * 1024; ///handle overflow final Stats stats; From 81c02c74c4c0bfb3ab930fafa470699282d45d8b Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 8 Jul 2022 12:33:53 +0300 Subject: [PATCH 37/99] Forwarding pointers done right + VerifyHeap hack HyperAlloc survives 1 minute! --- .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../core/genscavenge/ObjectHeaderImpl.java | 66 ++++++++++++------- .../oracle/svm/core/genscavenge/Space.java | 58 ++++++++++++---- .../oracle/svm/core/hub/LayoutEncoding.java | 6 ++ 4 files changed, 94 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5707069b1776..b9df0d6832ea 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -102,7 +102,6 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); - assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 6009ef596b8a..eca5f1aa732b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -327,19 +327,33 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { } } - private static final Unsafe UNSAFE = Unsafe.getUnsafe(); - /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") - static Object installForwardingPointer(Object original, Object copy) { - Log trace = Log.log(); - Word originalPtr = Word.objectToUntrackedPointer(original); -// assert !isPointerToForwardedObject(originalPtr); - UnsignedWord curHeader = readHeaderFromPointer(originalPtr); - if (isForwardedHeader(curHeader)) { - trace.string("PP reinstall for obj ").zhex(originalPtr).newline(); - return getForwardedObject(originalPtr, curHeader); + static void installForwardingPointer(Object original, Object copy) { + assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + UnsignedWord forwardHeader; + if (ReferenceAccess.singleton().haveCompressedReferences()) { + if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + } else { + forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + } + } else { + forwardHeader = Word.objectToUntrackedPointer(copy); } + assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); + writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + } + + private static final Unsafe UNSAFE = Unsafe.getUnsafe(); + + @AlwaysInline("GC performance") + static Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { + // create forwarding header UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { @@ -354,25 +368,27 @@ static Object installForwardingPointer(Object original, Object copy) { } else { forwardHeader = Word.objectToUntrackedPointer(copy); } - assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); -// writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - UnsignedWord newHeader = forwardHeader.or(FORWARDED_BIT); - boolean installed; + UnsignedWord bits = getHeaderBitsFromHeader(forwardHeader); + if (!bits.equal(0)) { + Log.log().string("OHI catch hdrbits in hdr ").unsigned(forwardHeader) + .string(" from obj ").object(copy).newline(); + } + forwardHeader = forwardHeader.or(FORWARDED_BIT); + + // try installing the new header + long witness; if (getReferenceSize() == Integer.BYTES) { -// ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); - int expected = (int) curHeader.rawValue(); - int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); - installed = witness == expected; + witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), + (int) originalHeader.rawValue(), (int) forwardHeader.rawValue()); } else { -// ObjectAccess.writeWord(original, getHubOffset(), newHeader); - long expected = curHeader.rawValue(); - long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); - installed = witness == expected; + witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), + originalHeader.rawValue(), forwardHeader.rawValue()); } + Pointer originalPtr = Word.objectToUntrackedPointer(original); assert isPointerToForwardedObject(originalPtr); - if (!installed) { - trace.string("PP collision for obj ").zhex(originalPtr).newline(); - return getForwardedObject(originalPtr); + if (witness != originalHeader.rawValue()) { + Log.log().string("PP collision for obj ").zhex(originalPtr).newline(); + return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0c662cf0cb6f..058d3770bef0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -31,9 +31,17 @@ import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.thread.Continuation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -376,36 +384,45 @@ Object promoteAlignedObject(Object original, Space originalSpace) { return copy; } - @AlwaysInline("GC performance") +/// @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Log trace = Log.log(); + ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); + Pointer originalMemory = Word.objectToUntrackedPointer(original); + UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { + trace.string("PP forward obj ").zhex(originalMemory).newline(); + return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); + } + + UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); + assert size.notEqual(0) : "zero obj size"; ///debug,rm + assert size.aboveThan(0) : "negative obj size"; Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + Log.log().string("SPC catch prob\n"); return null; } Object copy = copyMemory.toObject(); if (copy == null) { + Log.log().string("SPC catch null\n"); return null; } - // Read original header - Pointer originalMemory = Word.objectToUntrackedPointer(original); - assert ObjectHeaderImpl.getHubOffset() == 0; - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); // Install forwarding pointer into the original header - /// mv CAS code from OHI here - Object forward = ObjectHeaderImpl.installForwardingPointer(original, copy); + assert ObjectHeaderImpl.getHubOffset() == 0; ///PGC prerequisite + Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header + // We have won the race, now we must copy the object bits. First install the original originalHeader int headerSize = ObjectHeaderImpl.getReferenceSize(); if (headerSize == Integer.BYTES) { - copyMemory.writeInt(0, (int) header.rawValue()); + copyMemory.writeInt(0, (int) originalHeader.rawValue()); } else { - copyMemory.writeWord(0, header); + copyMemory.writeWord(0, originalHeader); } // copy the rest of original object UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); @@ -419,11 +436,30 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { ParallelGCImpl.queue(copyMemory); return copy; } else { + /// DEBUG: make a long array to make heap verifier happy + UnmanagedMemoryUtil.fillLongs(copyMemory, size, 0L); + DynamicHub fillClass = DynamicHub.fromClass(Object[].class); + Word fillHeader = ObjectHeaderImpl.encodeAsObjectHeader(fillClass, false, false); + int headerSize = ObjectHeaderImpl.getReferenceSize(); + copyMemory.writeWord(0, fillHeader); + copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); +// Log.log().string("PP fill ").zhex(copyMemory) +// .string(" len ").signed(size.rawValue() / headerSize - 2).newline(); +// Log.log().string(" hdr: ").zhex(copyMemory.readWord(0)) +// .string(" ").zhex(copyMemory.readWord(headerSize)) +// .newline(); + /// END DEBUG /// Now the allocated memory is lost. Retract in TLAB? return forward; } } + @AlwaysInline("GC performance") + public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header, ObjectHeaderImpl impl) { + int encoding = impl.dynamicHubFromObjectHeader(header).getLayoutEncoding(); + return LayoutEncoding.getSizeFromEncoding(obj, encoding); + } + @AlwaysInline("GC performance") private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 1a73be6c8d1d..b9b97cebfeb6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -269,6 +269,12 @@ public static UnsignedWord getSizeFromObject(Object obj) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInline(Object obj) { int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding(); + return getSizeFromEncoding(obj, encoding); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From db7c057bdda8af604cb06b7cbdfe7d1b93e53879 Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 14:37:09 +0300 Subject: [PATCH 38/99] Protected chunk promotion with mutex --- .../oracle/svm/core/genscavenge/Space.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 058d3770bef0..29490f6f037b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -494,8 +494,21 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); + originalSpace.mutex.lock(); + mutex.lock(); + try { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + } finally { + mutex.unlock(); + originalSpace.mutex.unlock(); + } + + if (ParallelGCImpl.isEnabled()) { + if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { + throw VMError.shouldNotReachHere(); + } + } if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { @@ -511,9 +524,15 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - Log.log().string("UA promote chunk ").zhex(chunk).newline(); - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); + originalSpace.mutex.lock(); + mutex.lock(); + try { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + } finally { + mutex.unlock(); + originalSpace.mutex.unlock(); + } if (ParallelGCImpl.isEnabled()) { if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { From 57d2372397648be3f518cb8f882dda7a9c790dd8 Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 15:33:40 +0300 Subject: [PATCH 39/99] Cleanup --- .../core/genscavenge/AlignedHeapChunk.java | 3 +- .../core/genscavenge/GreyObjectsWalker.java | 2 +- .../genscavenge/GreyToBlackObjectVisitor.java | 5 +- .../core/genscavenge/ObjectHeaderImpl.java | 29 +++--- .../oracle/svm/core/genscavenge/Space.java | 17 +--- .../core/genscavenge/UnalignedHeapChunk.java | 1 - .../svm/core/genscavenge/YoungGeneration.java | 7 -- .../genscavenge/parallel/ObjectQueue.java | 97 ------------------- .../core/genscavenge/parallel/TaskQueue.java | 12 ++- 9 files changed, 26 insertions(+), 147 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index b010ca3b9965..15a6eef96953 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -139,7 +138,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); } - @NeverInline("GC performance") + @AlwaysInline("GC performance") static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index d67cb5c9333e..366c20d0c9be 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -84,7 +84,7 @@ void walkGreyObjects() { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 4ec0d030e795..28f8e495d526 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; @@ -59,7 +58,7 @@ public boolean visitObject(Object o) { } @Override - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); @@ -69,7 +68,7 @@ public boolean visitObjectInline(Object o) { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index eca5f1aa732b..5d5673fa41f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -349,46 +349,41 @@ static void installForwardingPointer(Object original, Object copy) { assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); } - private static final Unsafe UNSAFE = Unsafe.getUnsafe(); - @AlwaysInline("GC performance") static Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { // create forwarding header UnsignedWord forwardHeader; + boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - assert false: "getCompressEncoding().hasShift()"; - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); ///non mt safe + hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); } } else { forwardHeader = Word.objectToUntrackedPointer(copy); } - UnsignedWord bits = getHeaderBitsFromHeader(forwardHeader); - if (!bits.equal(0)) { - Log.log().string("OHI catch hdrbits in hdr ").unsigned(forwardHeader) - .string(" from obj ").object(copy).newline(); - } + assert getHeaderBitsFromHeader(forwardHeader).equal(0); forwardHeader = forwardHeader.or(FORWARDED_BIT); // try installing the new header + Pointer originalPtr = Word.objectToUntrackedPointer(original); long witness; if (getReferenceSize() == Integer.BYTES) { - witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), - (int) originalHeader.rawValue(), (int) forwardHeader.rawValue()); + witness = originalPtr.compareAndSwapInt(getHubOffset(), + (int) originalHeader.rawValue(), (int) forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); } else { - witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), - originalHeader.rawValue(), forwardHeader.rawValue()); + witness = originalPtr.compareAndSwapLong(getHubOffset(), + originalHeader.rawValue(), forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); } - Pointer originalPtr = Word.objectToUntrackedPointer(original); assert isPointerToForwardedObject(originalPtr); if (witness != originalHeader.rawValue()) { - Log.log().string("PP collision for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); + } else if (hasShift) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 29490f6f037b..8e74f4401cfb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -187,12 +187,10 @@ Pointer allocateMemory(UnsignedWord objectSize) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } if (result.isNonNull()) { -// Log.log().string(" alloc fast ").zhex(result).newline(); return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); -// Log.log().string(" alloc slow ").zhex(result).newline(); return result; } finally { mutex.unlock(); @@ -384,31 +382,26 @@ Object promoteAlignedObject(Object original, Space originalSpace) { return copy; } -/// @AlwaysInline("GC performance") + @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - Log trace = Log.log(); ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); Pointer originalMemory = Word.objectToUntrackedPointer(original); UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { - trace.string("PP forward obj ").zhex(originalMemory).newline(); return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); - assert size.notEqual(0) : "zero obj size"; ///debug,rm - assert size.aboveThan(0) : "negative obj size"; + assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { - Log.log().string("SPC catch prob\n"); return null; } Object copy = copyMemory.toObject(); if (copy == null) { - Log.log().string("SPC catch null\n"); return null; } @@ -443,11 +436,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { int headerSize = ObjectHeaderImpl.getReferenceSize(); copyMemory.writeWord(0, fillHeader); copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); -// Log.log().string("PP fill ").zhex(copyMemory) -// .string(" len ").signed(size.rawValue() / headerSize - 2).newline(); -// Log.log().string(" hdr: ").zhex(copyMemory.readWord(0)) -// .string(" ").zhex(copyMemory.readWord(headerSize)) -// .newline(); /// END DEBUG /// Now the allocated memory is lost. Retract in TLAB? return forward; @@ -491,7 +479,6 @@ private Object copyAlignedObject(Object originalObj) { /** Promote an AlignedHeapChunk by moving it to this space. */ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { - /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); originalSpace.mutex.lock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index ad12df2e7cac..56843ba30cef 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -147,7 +147,6 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { @AlwaysInline("GC performance") public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { - Log.log().string("UA walk unaligned in chunk ").zhex(that).newline(); return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index e1215f39c62c..10f8d1dbb3e8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -145,13 +145,6 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { } void swapSpaces() { -// try (Log trace = Log.log()) { -// trace.string(">>> eden ind=").unsigned(getEden().getInd()).newline(); -// for (int i = 0; i < maxSurvivorSpaces; i++) { -// trace.string(">>> ss").unsigned(i).string(" ind=") -// .unsigned(getSurvivorFromSpaceAt(i).getInd()).newline(); -// } -// } for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorFromSpaceAt(i).isEmpty() : "Survivor fromSpace should be empty."; assert getSurvivorFromSpaceAt(i).getChunkBytes().equal(0) : "Chunk bytes must be 0"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java deleted file mode 100644 index b256b6805437..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -import java.util.function.BiConsumer; - -public class ObjectQueue {///rm - private final VMMutex mutex; - private final VMCondition cond; - private final Object[] item; - private boolean empty; - private volatile int idleCount; - - public ObjectQueue(String name) { - mutex = new VMMutex(name + "-queue"); - cond = new VMCondition(mutex); - item = new Object[2]; - empty = true; - } - - private Object[] get() { -// Log.log().string(" >>> OQ.get() called").newline(); - try { - mutex.lock(); - idleCount++; - while (empty) { -// Log.log().string(" >>> OQ.get() WAIT").newline(); - cond.block(); - } -// Log.log().string(" >>> OQ.get() returns").newline(); - return item; - } finally { - empty = true; - idleCount--; - mutex.unlock(); - cond.broadcast(); - } - } - - public void put(Object value0, Object value1) { -// Log.log().string(" >>> OQ.put() called").newline(); - try { - mutex.lock(); - while (!empty) { -// Log.log().string(" >>> OQ.put() WAIT").newline(); - cond.block(); - } - item[0] = value0; - item[1] = value1; -// Log.log().string(" >>> OQ.put() returns").newline(); - } finally { - empty = false; - mutex.unlock(); - cond.broadcast(); - } - } - - public void consume(BiConsumer consumer) { - Object val0, val1; -// Log.log().string(" >>> OQ.consume() called").newline(); - try { - mutex.lock(); - idleCount++; - while (empty) { -// Log.log().string(" >>> OQ.consume() WAIT").newline(); - cond.block(); - } -// Log.log().string(" >>> OQ.consume() unblocks").newline(); - val0 = item[0]; - val1 = item[1]; - } finally { - empty = true; - idleCount--; - mutex.unlock(); - cond.broadcast(); - } - consumer.accept(val0, val1); - } - - public void waitUntilIdle(int expectedIdleCount) { - try { - mutex.lock(); -// Log.log().string(">>> OQ.wait() empty=").bool(empty) -// .string(", idle=").signed(idleCount).newline(); - while (!empty) { - cond.block(); - } - } finally { - mutex.unlock(); - } - while (idleCount < expectedIdleCount); - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 519c784767bd..67f03a1326a3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -24,6 +24,10 @@ public TaskQueue(String name) { stats = new Stats(); } + private Log log() { + return Log.noopLog(); + } + private int next(int index) { return (index + 1) % SIZE; } @@ -40,7 +44,7 @@ public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { - Log.log().string("TQ cannot put task\n"); + log().string("TQ cannot put task\n"); cond.block(); } data[putIndex] = ptr; @@ -57,7 +61,7 @@ public void consume(Consumer consumer) { mutex.lock(); try { while (!canGet()) { - Log.log().string("TQ cannot get task\n"); + log().string("TQ cannot get task\n"); idleCount++; cond.block(); idleCount--; @@ -81,7 +85,7 @@ public void drain(Consumer consumer) { } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("TQ waitForIdle\n"); + log().string("TQ waitForIdle\n"); while (true) { try { mutex.lock(); @@ -89,7 +93,7 @@ public void waitUntilIdle(int expectedIdleCount) { cond.block(); } if (idleCount >= expectedIdleCount) { - log.string("TQ waitForIdle over\n"); + log().string("TQ waitForIdle over\n"); return; } } finally { From e376e13d4d416c41b7dac10e71cf930715b3b0fb Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 11 Jul 2022 10:58:30 +0300 Subject: [PATCH 40/99] Handle queue overflow by executing tasks synchronously --- .../genscavenge/parallel/ParallelGCImpl.java | 4 +++- .../core/genscavenge/parallel/TaskQueue.java | 17 +++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0374e9248a73..e3f224c7ed43 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,7 +53,9 @@ public static GreyToBlackObjectVisitor getVisitor() { } public static void queue(Pointer ptr) { - QUEUE.put(ptr); + if (!QUEUE.put(ptr)) { + PROMOTE_TASK.accept(ptr.toObject()); + } } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 67f03a1326a3..71aeaf0f5871 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -6,7 +6,7 @@ import org.graalvm.word.Pointer; public class TaskQueue { - private static final int SIZE = 1024 * 1024; ///handle overflow + private static final int SIZE = 128 * 1024; final Stats stats; @@ -40,20 +40,21 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(Pointer ptr) { + public boolean put(Pointer ptr) { try { mutex.lock(); - while (!canPut()) { + if (!canPut()) { log().string("TQ cannot put task\n"); - cond.block(); + return false; } data[putIndex] = ptr; - } finally { putIndex = next(putIndex); stats.noteTask(putIndex, getIndex); + } finally { mutex.unlock(); - cond.broadcast(); } + cond.broadcast(); + return true; } public void consume(Consumer consumer) { @@ -67,11 +68,11 @@ public void consume(Consumer consumer) { idleCount--; } ptr = data[getIndex]; - } finally { getIndex = next(getIndex); + } finally { mutex.unlock(); - cond.broadcast(); } + cond.broadcast(); consumer.accept(ptr.toObject()); } From 1c891d1023565dd7d8ef7560b0bbdbf936ccb689 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 11 Jul 2022 11:53:46 +0300 Subject: [PATCH 41/99] Cleanup --- substratevm/mx.substratevm/suite.py | 1 - .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 ++++---- .../svm/core/genscavenge/GreyObjectsWalker.java | 2 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 1 - .../svm/core/genscavenge/ObjectHeaderImpl.java | 2 -- .../svm/core/genscavenge/OldGeneration.java | 4 ---- .../genscavenge/ReferenceObjectProcessing.java | 4 +--- .../com/oracle/svm/core/genscavenge/Timers.java | 2 -- .../core/genscavenge/UnalignedHeapChunk.java | 1 - .../svm/core/genscavenge/YoungGeneration.java | 8 ++------ .../genscavenge/parallel/ParallelGCImpl.java | 2 -- .../core/genscavenge/parallel/TaskQueue.java | 6 +++--- .../oracle/svm/core/SubstrateDiagnostics.java | 7 ------- .../svm/core/hub/InteriorObjRefWalker.java | 6 +----- .../oracle/svm/core/stack/JavaStackWalker.java | 17 ----------------- .../com/oracle/svm/core/thread/VMThreads.java | 2 +- 16 files changed, 13 insertions(+), 60 deletions(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 342adf6795aa..f0eb593c9901 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -323,7 +323,6 @@ "requiresConcealed" : { "java.base": [ "sun.nio.ch", - "jdk.internal.misc", ], "java.management": [ "sun.management", diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 8fff237c339c..f6fd46b9745b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -866,7 +866,7 @@ private void blackenStackRoots() { JavaStackWalk walk = StackValue.get(JavaStackWalk.class); JavaStackWalker.initWalk(walk, sp, ip); - walkStack(walk, WordFactory.nullPointer()); + walkStack(walk); if (SubstrateOptions.MultiThreaded.getValue()) { /* @@ -884,7 +884,7 @@ private void blackenStackRoots() { continue; } if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk, vmThread); + walkStack(walk); } } } @@ -900,7 +900,7 @@ private void blackenStackRoots() { * calls to a stack frame visitor. */ @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) - private void walkStack(JavaStackWalk walk, IsolateThread thread) { + private void walkStack(JavaStackWalk walk) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; while (true) { @@ -913,7 +913,7 @@ private void walkStack(JavaStackWalk walk, IsolateThread thread) { DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); if (deoptFrame == null) { if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame, thread, walk.getAnchor(), walk.getIPCodeInfo(), codeInfo); + throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); } CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 366c20d0c9be..3788ec2572df 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -112,7 +112,7 @@ private void walkAlignedGreyObjects() { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 342b077c949e..d9d2b1ab8b31 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -106,7 +106,6 @@ public final class HeapImpl extends Heap { private final RuntimeCodeInfoGCSupportImpl runtimeCodeInfoGcSupport; private final ImageHeapInfo imageHeapInfo = new ImageHeapInfo(); private final HeapAccounting accounting = new HeapAccounting(); - public final Timer timer = new Timer("promoteUnalignedHeapChunk()"); /** Head of the linked list of currently pending (ready to be enqueued) {@link Reference}s. */ private Reference refPendingList; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 5d5673fa41f3..345a32a91f6b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.log.Log; -import jdk.internal.misc.Unsafe; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.CompressEncoding; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index f6f2c9d9f24a..8b3e042e34f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -27,7 +27,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; @@ -108,13 +107,10 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[OldGeneration.scanGreyObjects:").newline(); if (!toGreyObjectsWalker.haveGreyObjects()) { - trace.string("]").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); - trace.string("]").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 7923285d6e2a..3c0a614336fb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -31,7 +31,6 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; -import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -78,7 +77,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { softReferencesAreWeak = enabled; } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,7 +86,6 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } - @NeverInline("GC performance") private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index e00701d54eaf..bd5b1cd255fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -150,7 +150,6 @@ void resetAllExceptMutator() { referenceObjects.reset(); releaseSpaces.reset(); verifyAfter.reset(); - HeapImpl.getHeapImpl().timer.reset(); /* The mutator timer is *not* reset here. */ trace.string("]").newline(); } @@ -172,7 +171,6 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", blackenImageHeapRoots); logOneTimer(log, " ", blackenDirtyCardRoots); logOneTimer(log, " ", scanGreyObjects); - logOneTimer(log, " ", HeapImpl.getHeapImpl().timer); logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); logOneTimer(log, " ", releaseSpaces); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 56843ba30cef..299309cb970b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 10f8d1dbb3e8..209fbad2731d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -136,10 +136,8 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { } void releaseSpaces(ChunkReleaser chunkReleaser) { -// getEden().setInd(333); getEden().releaseChunks(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { -// getSurvivorFromSpaceAt(i).setInd(100+i); getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); } } @@ -173,7 +171,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:").newline(); + Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -182,13 +180,11 @@ boolean scanGreyObjects() { } } if (!needScan) { - trace.string("]").newline(); return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).newline(); + trace.string("[Scanning survivor-").signed(i).string("]").newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); - trace.string("]").newline(); } trace.string("]").newline(); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e3f224c7ed43..cca0a7acae96 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -30,8 +30,6 @@ public void startWorkerThreads() { } public void startWorkerThread(int n) { - final Log trace = Log.log(); - trace.string("PP start worker-").unsigned(n).newline(); Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 71aeaf0f5871..89a3ab4a3170 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -12,7 +12,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private final Pointer[] data; /// move out of heap? + private final Pointer[] data; private int idleCount; private int getIndex; private int putIndex; @@ -103,8 +103,8 @@ public void waitUntilIdle(int expectedIdleCount) { } } - public interface Consumer { - void accept(Object object); ///j.u.f.Consumer + interface Consumer { + void accept(Object object); } // Non MT safe, needs locking diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index 1c9dd8d108d4..d0b83fa3ace9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -712,13 +712,6 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } log.string(", stack(").zhex(VMThreads.StackEnd.get(thread)).string(",").zhex(VMThreads.StackBase.get(thread)).string(")"); - JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); - if (anchor.isNonNull()) { - log.newline().spaces(21) - .string("anchor=").zhex(anchor) - .string(", sp=").zhex(anchor.getLastJavaSP()) - .string(", ip=").zhex(anchor.getLastJavaIP()); - } log.newline(); printed++; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index bc5dde38d981..0d4ae77e5d2f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.hub; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.word.Word; import org.graalvm.word.Pointer; @@ -61,14 +60,11 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor return walkObjectInline(obj, visitor); } - @NeverInline("Performance critical version") + @AlwaysInline("Performance critical version") public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); - if (objHub.getName() == null) { - Log.log().string("PP warn hub.name=null for obj ").zhex(objPointer).newline(); - } // Visit each Object reference in the array part of the Object. if (LayoutEncoding.isArrayLikeWithObjectElements(layoutEncoding)) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index c9bc20c9e0b6..96bfa38324f8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -245,23 +245,6 @@ public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePoi } - @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) - public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame, - IsolateThread thread, JavaFrameAnchor anchor, UntetheredCodeInfo uCodeInfo, CodeInfo codeInfo) - { - Log log = Log.log().string("Stack walk must walk only frames of known code:"); - log.string(" anchor=").hex(anchor).string(" sp=").hex(sp).string(" ip=").hex(ip).newline() - .string(" thread=").hex(thread).string(" codeInfo=").hex(codeInfo) - .string(" untethered=").hex(uCodeInfo); - - if (DeoptimizationSupport.enabled()) { - log.string(" deoptFrame=").object(deoptFrame); - } - log.newline(); - throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); - - } - public static boolean walkCurrentThread(Pointer startSP, StackFrameVisitor visitor) { return walkCurrentThread(startSP, visitor, null); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index f10e1ae9aa94..49fcd616faaa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,7 +826,7 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } - public static class ParallelGCSupport { + public static class ParallelGCSupport {///unused private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); private static final int REGULAR_THREAD = 0; private static final int PARALLEL_GC_THREAD = 1; From 7b880b63404025bde7acf3583ef2992d972a4b3a Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 19 Jul 2022 18:40:01 +0300 Subject: [PATCH 42/99] Added thread local Stats --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 +- .../genscavenge/parallel/ParallelGCImpl.java | 13 +- .../svm/core/genscavenge/parallel/Stats.java | 204 ++++++++++++++++++ .../core/genscavenge/parallel/TaskQueue.java | 32 --- 4 files changed, 215 insertions(+), 39 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f6fd46b9745b..64334259017b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,6 +32,7 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.Stats; import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; @@ -442,9 +443,7 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - TaskQueue.Stats stats = ParallelGCImpl.getStats(); - verboseGCLog.newline().string(" tasks: ").unsigned(stats.getCount()) - .string(", max size: ").unsigned(stats.getMaxSize()).newline(); + ParallelGCImpl.getStats().print(Log.log()); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index cca0a7acae96..e6aa63d03c51 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -33,6 +33,7 @@ public void startWorkerThread(int n) { Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); + getStats().install(); try { while (!stopped) { QUEUE.consume(PROMOTE_TASK); @@ -72,10 +73,6 @@ public static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } - public static TaskQueue.Stats getStats() { - return QUEUE.stats; - } - public static void checkThrowable() { if (throwable != null) { Log.log().string("PGC error : ").string(throwable.getClass().getName()) @@ -84,6 +81,14 @@ public static void checkThrowable() { throw new Error(throwable); } } + + public static Stats getStats() { + return Stats.stats(); + } + + static Log log() { + return Log.noopLog(); + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java new file mode 100644 index 000000000000..a08cbd0c1ae7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java @@ -0,0 +1,204 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; +import org.graalvm.nativeimage.IsolateThread; + +public abstract class Stats { + private static final Stats realStats = new RealStats(); + private static final Stats noopStats = new NoopStats(); + + static Stats stats() { + return realStats; + } + + static Stats noopStats() { + return noopStats; + } + + public void reset() {} + public void print(Log log) {} + void install() {} + public void noteAllocTime(long t) {} + public void noteForwardInstallTime(long t) {} + public void noteObjectVisitTime(long t) {} + public void noteQueueCopyTime(long t) {} + public void noteQueueScanTime(long t) {} + public void noteLostObject() {} + void notePut(int putIndex, int getIndex, int capacity) {} + void noteGet() {} +} + +class NoopStats extends Stats { +} + +class RealStats extends Stats { + private static final FastThreadLocalObject statsTL = + FastThreadLocalFactory.createObject(StatsImpl.class, "ParGC.Stats"); + + private final StatsImpl defaultStats = new StatsImpl(); + private final StatsImpl totalStats = new StatsImpl(); + + private StatsImpl impl() { + StatsImpl stats = statsTL.get(); + if (stats == null) { + stats = defaultStats; + statsTL.set(stats); + } + return stats; + } + + @Override + void install() { + statsTL.set(new StatsImpl()); + } + + @Override + public void reset() { + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + StatsImpl stats = statsTL.get(vmThread); + if (stats != null) { + stats.reset(); + } + } + } + + @Override + public void print(Log log) { + totalStats.reset(); + log.string("PGC stats:").newline(); + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + StatsImpl stats = statsTL.get(vmThread); + if (stats != null) { + Thread jt = PlatformThreads.fromVMThread(vmThread); + log.string(" ").string(jt.getName()).string(":").newline(); + totalStats.objectVisitTime += stats.objectVisitTime; + totalStats.allocTime += stats.allocTime; + totalStats.forwardInstallTime += stats.forwardInstallTime; + totalStats.queueScanTime += stats.queueScanTime; + totalStats.queueCopyTime += stats.queueCopyTime; + totalStats.lostObjectCount += stats.lostObjectCount; + totalStats.putCount += stats.putCount; + totalStats.getCount += stats.getCount; + totalStats.maxQueueSize = Math.max(totalStats.maxQueueSize, stats.maxQueueSize); + stats.print(log); + } + } + log.string(" total:").newline(); + totalStats.print(log); + } + + @Override + public void noteAllocTime(long t) { + impl().noteAllocTime(t); + } + + @Override + public void noteForwardInstallTime(long t) { + impl().noteForwardInstallTime(t); + } + + @Override + public void noteObjectVisitTime(long t) { + impl().noteObjectVisitTime(t); + } + + @Override + public void noteQueueCopyTime(long t) { + impl().noteQueueCopyTime(t); + } + + @Override + public void noteQueueScanTime(long t) { + impl().noteQueueScanTime(t); + } + + @Override + public void noteLostObject() { + impl().noteLostObject(); + } + + @Override + void notePut(int putIndex, int getIndex, int capacity) { + impl().notePut(putIndex, getIndex, capacity); + } + + @Override + public void noteGet() { + impl().noteGet(); + } +} + +class StatsImpl extends Stats { + int allocTime, forwardInstallTime, objectVisitTime, queueCopyTime, queueScanTime; + int putCount, getCount, lostObjectCount, maxQueueSize; + + @Override + public void reset() { + allocTime = forwardInstallTime = objectVisitTime = queueCopyTime = queueScanTime = 0; + putCount = getCount = lostObjectCount = maxQueueSize = 0; + } + + @Override + public void print(Log log) { + log.string(" visit ").unsigned(objectVisitTime) + .string(" alloc ").unsigned(allocTime) + .string(" fwptr ").unsigned(forwardInstallTime) + .string(" qscan ").unsigned(queueScanTime) + .string(" qcopy ").unsigned(queueCopyTime) + .string(" lost ").unsigned(lostObjectCount).newline(); + log.string(" put ").unsigned(putCount) + .string(" get ").unsigned(getCount) + .string(" max ").unsigned(maxQueueSize).newline(); + } + + @Override + public void noteAllocTime(long t) { + allocTime += t; + } + + @Override + public void noteForwardInstallTime(long t) { + forwardInstallTime += t; + } + + @Override + public void noteObjectVisitTime(long t) { + objectVisitTime += t; + } + + @Override + public void noteQueueCopyTime(long t) { + queueCopyTime += t; + } + + @Override + public void noteQueueScanTime(long t) { + queueScanTime += t; + } + + @Override + public void noteLostObject() { + lostObjectCount++; + } + + @Override + void notePut(int putIndex, int getIndex, int capacity) { + int size = putIndex - getIndex; + if (size < 0) { + size += capacity; + } + if (size > maxQueueSize) { + maxQueueSize = size; + } + putCount++; + } + + @Override + void noteGet() { + getCount++; + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 89a3ab4a3170..aaafd18fdea6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,8 +8,6 @@ public class TaskQueue { private static final int SIZE = 128 * 1024; - final Stats stats; - private final VMMutex mutex; private final VMCondition cond; private final Pointer[] data; @@ -21,7 +19,6 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = new Pointer[SIZE]; - stats = new Stats(); } private Log log() { @@ -49,7 +46,6 @@ public boolean put(Pointer ptr) { } data[putIndex] = ptr; putIndex = next(putIndex); - stats.noteTask(putIndex, getIndex); } finally { mutex.unlock(); } @@ -106,32 +102,4 @@ public void waitUntilIdle(int expectedIdleCount) { interface Consumer { void accept(Object object); } - - // Non MT safe, needs locking - public static class Stats { - private int count, maxSize; - - void noteTask(int putIndex, int getIndex) { - int size = putIndex - getIndex; - if (size < 0) { - size += SIZE; - } - if (size > maxSize) { - maxSize = size; - } - count++; - } - - public int getCount() { - return count; - } - - public int getMaxSize() { - return maxSize; - } - - public void reset() { - count = maxSize = 0; - } - } } \ No newline at end of file From cdf48155055b1327e4e548803b8e4aad41631218 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 18 Jul 2022 11:30:47 +0300 Subject: [PATCH 43/99] (Incomplete) CAS based queue with minimal synchronization. --- .../core/genscavenge/parallel/CasQueue.java | 85 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 38 +++++++-- 2 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java new file mode 100644 index 000000000000..107fedc3afbd --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java @@ -0,0 +1,85 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +import java.util.concurrent.atomic.AtomicInteger; + +/// Lame, test only. Doesn't handle overflows +public class CasQueue { + private static final int SIZE = 1024 * 1024; + + final Stats stats = new Stats(); + + private final Pointer[] data = new Pointer[SIZE]; + private final AtomicInteger getIndex = new AtomicInteger(0); + private final AtomicInteger putIndex = new AtomicInteger(0); + + private Log log() { + return ParallelGCImpl.log(); + } + + public boolean put(Pointer ptr) { + int cur = putIndex.get(); + int witness; + while ((witness = putIndex.compareAndExchange(cur, cur + 1)) != cur) { + cur = witness; + } + data[cur] = ptr; + return true; + } + + public boolean consume(Consumer consumer) { + int cur = getIndex.get(); + int witness; + while (cur < putIndex.get() && (witness = getIndex.compareAndExchange(cur, cur + 1)) != cur) { + cur = witness; + } + if (cur >= putIndex.get()) { + return false; + } + Object obj = data[cur].toObject(); + consumer.accept(obj); + return true; + } + + public void drain(Consumer consumer) { + while (consume(consumer)); + getIndex.set(0); + putIndex.set(0); + } + + interface Consumer { + void accept(Object object); + } + + // Non MT safe, needs locking + public static class Stats { + private int count, maxSize; + + void noteTask(int putIndex, int getIndex) { + int size = putIndex - getIndex; + if (size < 0) { + size += SIZE; + } + if (size > maxSize) { + maxSize = size; + } + count++; + } + + public int getCount() { + return count; + } + + public int getMaxSize() { + return maxSize; + } + + public void reset() { + count = maxSize = 0; + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e6aa63d03c51..6f7812a32243 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -4,12 +4,15 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -17,8 +20,12 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - private static final TaskQueue QUEUE = new TaskQueue("pargc"); - private static final TaskQueue.Consumer PROMOTE_TASK = + private static final VMMutex mutex = new VMMutex("pargc"); + private static final VMCondition cond = new VMCondition(mutex); + private static AtomicInteger busy = new AtomicInteger(0); + + private static final CasQueue QUEUE = new CasQueue(); + private static final CasQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); private static volatile boolean enabled; @@ -33,10 +40,22 @@ public void startWorkerThread(int n) { Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); + log().string("WW start ").unsigned(n).newline(); getStats().install(); try { while (!stopped) { - QUEUE.consume(PROMOTE_TASK); + mutex.lock(); + while (busy.get() < WORKERS_COUNT) { + log().string("WW block ").unsigned(n).newline(); + cond.block(); + } + mutex.unlock(); + + QUEUE.drain(PROMOTE_TASK); + if (busy.decrementAndGet() <= 0) { + cond.broadcast(); + } + log().string("WW idle ").unsigned(n).newline(); } } catch (Throwable e) { throwable = e; @@ -58,11 +77,16 @@ public static void queue(Pointer ptr) { } public static void waitForIdle() { - if (WORKERS_COUNT > 0) { - QUEUE.waitUntilIdle(WORKERS_COUNT); - } else { - QUEUE.drain(PROMOTE_TASK); // execute synchronously + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle } + mutex.unlock(); } public static boolean isEnabled() { From 71c515a851ed9f0b4c221045450bf1db5b40930a Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 19 Jul 2022 12:20:05 +0300 Subject: [PATCH 44/99] Thread local memory allocation --- .../oracle/svm/core/genscavenge/GCImpl.java | 1 + .../oracle/svm/core/genscavenge/Space.java | 21 ++++++++++--------- .../genscavenge/parallel/ParallelGCImpl.java | 9 ++++++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 64334259017b..6761a8f28b84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1157,6 +1157,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8e74f4401cfb..df7cd6ce9a16 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -178,18 +178,18 @@ public Log report(Log log, boolean traceHeapChunks) { */ @NeverInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the last chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + } + if (result.isNonNull()) { + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ mutex.lock(); try { - Pointer result = WordFactory.nullPointer(); - /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); - if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { - return result; - } - /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); return result; } finally { @@ -199,6 +199,7 @@ Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + ParallelGCImpl.TLAB.set(newChunk); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 6f7812a32243..8a7afa9a68af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; @@ -8,9 +9,12 @@ import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; @@ -20,6 +24,10 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; + /// tlab of the current to-space + public static final FastThreadLocalWord TLAB = + FastThreadLocalFactory.createWord("ParallelGCImpl.TLAB"); + private static final VMMutex mutex = new VMMutex("pargc"); private static final VMCondition cond = new VMCondition(mutex); private static AtomicInteger busy = new AtomicInteger(0); @@ -52,6 +60,7 @@ public void startWorkerThread(int n) { mutex.unlock(); QUEUE.drain(PROMOTE_TASK); + TLAB.set(WordFactory.nullPointer()); if (busy.decrementAndGet() <= 0) { cond.broadcast(); } From 1d2f791087cb192514708b0e59e657cb8d8fa503 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 8 Aug 2022 18:08:07 +0300 Subject: [PATCH 45/99] Retract speculatively allocated memory --- .../core/genscavenge/AlignedHeapChunk.java | 8 +++++ .../oracle/svm/core/genscavenge/Space.java | 29 +++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 15a6eef96953..8d81707a7293 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -114,6 +114,14 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } + /** Retract last allocation made. Used by parallel collector. */ + static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { + Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); + assert newTop.aboveOrEqual(HeapChunk.asPointer(that)); + HeapChunk.setTopPointer(that, newTop); + return newTop; + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static UnsignedWord getCommittedObjectMemory(AlignedHeader that) { return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index df7cd6ce9a16..afa6519f0bb9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -176,7 +176,7 @@ public Log report(Log log, boolean traceHeapChunks) { /** * Allocate memory from an AlignedHeapChunk in this Space. */ - @NeverInline("GC performance") + @AlwaysInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ @@ -197,6 +197,16 @@ Pointer allocateMemory(UnsignedWord objectSize) { } } + /** + * Retract last allocation made. Used by parallel collector. + */ + @AlwaysInline("GC performance") + Pointer freeMemory(UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + assert oldChunk.isNonNull(); + return AlignedHeapChunk.freeMemory(oldChunk, objectSize); + } + private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); ParallelGCImpl.TLAB.set(newChunk); @@ -395,6 +405,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } + // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. + // If another thread copies the object first, we retract the allocation later. UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); @@ -411,14 +423,14 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original originalHeader + // We have won the race, now we must copy the object bits. First install the original header int headerSize = ObjectHeaderImpl.getReferenceSize(); if (headerSize == Integer.BYTES) { copyMemory.writeInt(0, (int) originalHeader.rawValue()); } else { copyMemory.writeWord(0, originalHeader); } - // copy the rest of original object + // Copy the rest of original object UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); if (isOldSpace()) { @@ -430,15 +442,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { ParallelGCImpl.queue(copyMemory); return copy; } else { - /// DEBUG: make a long array to make heap verifier happy - UnmanagedMemoryUtil.fillLongs(copyMemory, size, 0L); - DynamicHub fillClass = DynamicHub.fromClass(Object[].class); - Word fillHeader = ObjectHeaderImpl.encodeAsObjectHeader(fillClass, false, false); - int headerSize = ObjectHeaderImpl.getReferenceSize(); - copyMemory.writeWord(0, fillHeader); - copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); - /// END DEBUG - /// Now the allocated memory is lost. Retract in TLAB? + // Retract speculatively allocated memory + freeMemory(size); return forward; } } From 17c695bbe59466ed3f7bbee745104586dda07879 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 10 Aug 2022 12:33:49 +0300 Subject: [PATCH 46/99] Thread local task stacks --- .../oracle/svm/core/genscavenge/GCImpl.java | 18 ++++----- .../genscavenge/GreyToBlackObjectVisitor.java | 4 +- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 32 ++++++++++++---- .../parallel/ThreadLocalTaskStack.java | 37 +++++++++++++++++++ 5 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 6761a8f28b84..1d41c97d9063 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1055,17 +1055,15 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - ParallelGCImpl.setEnabled(true); - try { - if (isIncremental) { - heap.getYoungGeneration().scanGreyObjects(); - } - heap.getOldGeneration().scanGreyObjects(); - } finally { - ParallelGCImpl.waitForIdle(); - ParallelGCImpl.setEnabled(false); - scanGreyObjectsTimer.close(); + if (isIncremental) { + heap.getYoungGeneration().scanGreyObjects(); } + heap.getOldGeneration().scanGreyObjects(); + + ParallelGCImpl.setEnabled(true); + ParallelGCImpl.waitForIdle(); + ParallelGCImpl.setEnabled(false); + scanGreyObjectsTimer.close(); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 28f8e495d526..b32f1ee828c0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -60,8 +60,8 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); + if (!ParallelGCImpl.isEnabled()) { + ParallelGCImpl.push(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index afa6519f0bb9..8d4b7f8c9674 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -439,7 +439,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.queue(copyMemory); + ParallelGCImpl.localPush(copyMemory); return copy; } else { // Retract speculatively allocated memory diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8a7afa9a68af..859ca460775a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -32,9 +32,10 @@ public class ParallelGCImpl extends ParallelGC { private static final VMCondition cond = new VMCondition(mutex); private static AtomicInteger busy = new AtomicInteger(0); - private static final CasQueue QUEUE = new CasQueue(); - private static final CasQueue.Consumer PROMOTE_TASK = - obj -> getVisitor().doVisitObject(obj); + private static final ThreadLocalTaskStack[] STACKS = + IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); + private static final ThreadLocal localStack = new ThreadLocal<>(); + private static int currentStack; private static volatile boolean enabled; private static volatile Throwable throwable; @@ -49,6 +50,9 @@ public void startWorkerThread(int n) { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); + + final ThreadLocalTaskStack stack = STACKS[n]; + localStack.set(STACKS[n]); getStats().install(); try { while (!stopped) { @@ -59,7 +63,11 @@ public void startWorkerThread(int n) { } mutex.unlock(); - QUEUE.drain(PROMOTE_TASK); + log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); + Object obj; + while ((obj = stack.pop()) != null) { + getVisitor().doVisitObject(obj); + } TLAB.set(WordFactory.nullPointer()); if (busy.decrementAndGet() <= 0) { cond.broadcast(); @@ -79,9 +87,19 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static void queue(Pointer ptr) { - if (!QUEUE.put(ptr)) { - PROMOTE_TASK.accept(ptr.toObject()); + /// name, explain + public static void push(Pointer ptr) { + push(ptr, STACKS[currentStack]); + currentStack = (currentStack + 1) % WORKERS_COUNT; + } + + public static void localPush(Pointer ptr) { + push(ptr, localStack.get()); + } + + private static void push(Pointer ptr, ThreadLocalTaskStack stack) { + if (!stack.push(ptr)) { + getVisitor().doVisitObject(ptr.toObject()); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java new file mode 100644 index 000000000000..09645ebcc404 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java @@ -0,0 +1,37 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +/// thread local, non MT safe +public class ThreadLocalTaskStack { + private static final int SIZE = 128 * 1024; + + private final Pointer[] data = new Pointer[SIZE]; + private int top = 0; + + private Log log() { + return ParallelGCImpl.log(); + } + + boolean push(Pointer ptr) { + if (top >= SIZE) { + log().string("TT cannot put task\n"); + return false; + } + data[top++] = ptr; + return true; + } + + Object pop() { + if (top > 0) { + return data[--top].toObjectNonNull(); + } else { + return null; + } + } + + int size() { + return top; + } +} \ No newline at end of file From d9a47b7075b9a6471567a24e2838ee3034dc2cfc Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 12 Aug 2022 10:25:20 +0300 Subject: [PATCH 47/99] Post merge cleanup --- .../oracle/svm/core/genscavenge/GCImpl.java | 14 ++------ .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../genscavenge/GreyToBlackObjectVisitor.java | 2 -- .../svm/core/genscavenge/HeapChunk.java | 1 - .../oracle/svm/core/genscavenge/Space.java | 12 +------ .../genscavenge/parallel/ParallelGCImpl.java | 1 - .../com/oracle/svm/core/heap/ParallelGC.java | 33 ------------------- .../com/oracle/svm/core/thread/VMThreads.java | 14 -------- 8 files changed, 4 insertions(+), 74 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1d41c97d9063..a8cb29389656 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -26,20 +26,12 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.genscavenge.parallel.Stats; -import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.heap.VMOperationInfos; -import com.oracle.svm.core.hub.LayoutEncoding; -import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -377,10 +369,9 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); - boolean compressed = ReferenceAccess.singleton().haveCompressedReferences(); + boolean compressed = ReferenceAccess.singleton().haveCompressedReferences();///rm boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); - verboseGCLog.string("]").newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } @@ -1052,6 +1043,7 @@ private static void prepareForPromotion(boolean isIncremental) { } } + /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); @@ -1155,7 +1147,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); + ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); ///rm? } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index b9df0d6832ea..34637ae2b532 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index b32f1ee828c0..ccce076078a1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,8 +25,6 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import org.graalvm.compiler.nodes.java.ArrayLengthNode; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index a81495658463..f56f97261d9c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,7 +26,6 @@ import java.util.function.IntUnaryOperator; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8d4b7f8c9674..3d5b61d00ce9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -27,21 +27,10 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AlwaysInline; -import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.heap.StoredContinuation; -import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.thread.Continuation; -import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; -import org.graalvm.compiler.nodes.java.ArrayLengthNode; -import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -234,6 +223,7 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ ///assert owns mutex || pargc thread ? + ///use thread mutex to protect appendNewChunk // if (SubstrateOptions.MultiThreaded.getValue()) { // VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); // } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 859ca460775a..c46d32d6ecff 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -47,7 +47,6 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { Thread t = new Thread(() -> { -// VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 0275910defab..c62182edba9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,40 +1,7 @@ package com.oracle.svm.core.heap; -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; - public abstract class ParallelGC { - protected final VMMutex mutex = new VMMutex("pargc"); - protected final VMCondition cond = new VMCondition(mutex); - - protected volatile boolean workerThreadActive; protected volatile boolean stopped; public abstract void startWorkerThreads(); - - public void waitForNotification(boolean onWorkerThread) { - Log trace = Log.log(); - mutex.lock(); - try { - while (workerThreadActive != onWorkerThread) { - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - cond.block(); - trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); - } - } finally { - mutex.unlock(); - } - } - - public void signal(boolean onWorkerThread) { - mutex.lock(); - try { - Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - workerThreadActive = !onWorkerThread; - cond.broadcast(); - } finally { - mutex.unlock(); - } - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 49fcd616faaa..9a4b14251f57 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,20 +826,6 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } - public static class ParallelGCSupport {///unused - private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); - private static final int REGULAR_THREAD = 0; - private static final int PARALLEL_GC_THREAD = 1; - - public static boolean isParallelGCThread(IsolateThread vmThread) { - return isParallelGCThreadTL.getVolatile(vmThread) == PARALLEL_GC_THREAD; - } - - public static void setParallelGCThread() { - isParallelGCThreadTL.setVolatile(PARALLEL_GC_THREAD); - } - } - public static class SafepointBehavior { /** Determines how this thread interacts with the safepoint handling. */ private static final FastThreadLocalInt safepointBehaviorTL = FastThreadLocalFactory.createInt("StatusSupport.safepointBehaviorTL"); From ef9c35f7eca5c2fa450c35a020b30837051f122f Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 15 Aug 2022 17:04:43 +0300 Subject: [PATCH 48/99] Introduced `UseParallelGC` option --- .../core/genscavenge/AlignedHeapChunk.java | 6 +-- .../oracle/svm/core/genscavenge/GCImpl.java | 19 ++++---- .../svm/core/genscavenge/HeapChunk.java | 3 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../core/genscavenge/PinnedObjectImpl.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 29 +++++------- .../core/genscavenge/UnalignedHeapChunk.java | 2 +- ...rialOrEpsilonGC.java => UseGraalCeGC.java} | 4 +- .../graal/GenScavengeGCFeature.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 45 ++++++++++++------- .../com/oracle/svm/core/SubstrateOptions.java | 26 +++++++++++ .../graal/snippets/CEntryPointSnippets.java | 2 +- .../com/oracle/svm/core/heap/ParallelGC.java | 26 ++++++++++- .../oracle/svm/hosted/heap/PodSupport.java | 2 +- 14 files changed, 114 insertions(+), 56 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{UseSerialOrEpsilonGC.java => UseGraalCeGC.java} (90%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 8d81707a7293..25f25d0a653d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -114,10 +114,10 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } - /** Retract last allocation made. Used by parallel collector. */ + /** Retract the latest allocation. Used by parallel collector. */ static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); - assert newTop.aboveOrEqual(HeapChunk.asPointer(that)); + assert newTop.aboveThan(HeapChunk.asPointer(that)); HeapChunk.setTopPointer(that, newTop); return newTop; } @@ -184,7 +184,7 @@ public UnsignedWord getAllocationStart(AlignedHeapChunk.AlignedHeader heapChunk) class AlignedHeapChunkMemoryWalkerAccessFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index a8cb29389656..ee38df73b794 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,6 +30,7 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; @@ -1045,17 +1046,17 @@ private static void prepareForPromotion(boolean isIncremental) { /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { - HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - if (isIncremental) { - heap.getYoungGeneration().scanGreyObjects(); + try { + if (isIncremental) { + scanGreyObjectsLoop(); + } else { + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + ParallelGCImpl.waitForIdle(); + } + } finally { + scanGreyObjectsTimer.close(); } - heap.getOldGeneration().scanGreyObjects(); - - ParallelGCImpl.setEnabled(true); - ParallelGCImpl.waitForIdle(); - ParallelGCImpl.setEnabled(false); - scanGreyObjectsTimer.close(); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index f56f97261d9c..962423f5c1f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -282,8 +282,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; - Pointer top = getTopPointer(that); - while (offset.belowThan(top)) { + while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read Object obj = offset.toObject(); if (!visitor.visitObjectInline(obj)) { return false; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index d9d2b1ab8b31..57d9da3f677a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -887,7 +887,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseSerialOrEpsilonGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseGraalCeGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index 0622b3561bd3..1047bf0ff49d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -69,7 +69,7 @@ public boolean isPinned(Object object) { static class PinnedObjectFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3d5b61d00ce9..bebbe655e344 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -39,6 +39,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; @@ -47,6 +48,7 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; /** * A Space is a collection of HeapChunks. @@ -166,7 +168,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") - Pointer allocateMemory(UnsignedWord objectSize) { + private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); @@ -187,10 +189,10 @@ Pointer allocateMemory(UnsignedWord objectSize) { } /** - * Retract last allocation made. Used by parallel collector. + * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - Pointer freeMemory(UnsignedWord objectSize) { + private Pointer freeMemory(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); @@ -222,11 +224,9 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - ///assert owns mutex || pargc thread ? - ///use thread mutex to protect appendNewChunk -// if (SubstrateOptions.MultiThreaded.getValue()) { -// VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); -// } + if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); + } appendAlignedHeapChunkUninterruptibly(aChunk); accounting.noteAlignedHeapChunk(); } @@ -277,10 +277,9 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - ///assert owns mutex || pargc thread ? -// if (SubstrateOptions.MultiThreaded.getValue()) { -// VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); -// } + if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); + } appendUnalignedHeapChunkUninterruptibly(uChunk); accounting.noteUnalignedHeapChunk(uChunk); } @@ -409,7 +408,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } // Install forwarding pointer into the original header - assert ObjectHeaderImpl.getHubOffset() == 0; ///PGC prerequisite + assert ObjectHeaderImpl.getHubOffset() == 0; Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { @@ -477,14 +476,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.mutex.lock(); mutex.lock(); try { originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); } finally { mutex.unlock(); - originalSpace.mutex.unlock(); } if (ParallelGCImpl.isEnabled()) { @@ -507,14 +504,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.mutex.lock(); mutex.lock(); try { originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); } finally { mutex.unlock(); - originalSpace.mutex.unlock(); } if (ParallelGCImpl.isEnabled()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 299309cb970b..30014b36b163 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -186,7 +186,7 @@ public UnsignedWord getAllocationStart(UnalignedHeapChunk.UnalignedHeader heapCh class UnalignedHeapChunkMemoryWalkerAccessFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java similarity index 90% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java index cce0bb85bf71..47edd9b1ee6d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java @@ -32,9 +32,9 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -class UseSerialOrEpsilonGC implements BooleanSupplier { +class UseGraalCeGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index 61569d837a40..7fe017b4a49d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -63,7 +63,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index c46d32d6ecff..c89638509b14 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,8 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -11,6 +13,7 @@ import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -42,6 +45,10 @@ public class ParallelGCImpl extends ParallelGC { @Override public void startWorkerThreads() { + /// determine worker count + int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); + VMError.guarantee(hubOffset == 0, "hub offset must be 0"); + IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); } @@ -54,7 +61,7 @@ public void startWorkerThread(int n) { localStack.set(STACKS[n]); getStats().install(); try { - while (!stopped) { + while (true) { mutex.lock(); while (busy.get() < WORKERS_COUNT) { log().string("WW block ").unsigned(n).newline(); @@ -103,23 +110,29 @@ private static void push(Pointer ptr, ThreadLocalTaskStack stack) { } public static void waitForIdle() { - log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run - - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle + if (isSupported()) { + setEnabled(true); + + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle + } + mutex.unlock(); + + setEnabled(false); } - mutex.unlock(); } public static boolean isEnabled() { return enabled; } - public static void setEnabled(boolean enabled) { + private static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } @@ -143,11 +156,11 @@ static Log log() { @AutomaticFeature class ParallelGCFeature implements Feature { -/// -// @Override -// public boolean isInConfiguration(IsInConfigurationAccess access) { -// return SubstrateOptions.UseSerialGC.getValue(); -// } + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return SubstrateOptions.UseParallelGC.getValue(); + } @Override public void afterRegistration(AfterRegistrationAccess access) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index d125b205cf10..94ff01a0fd98 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -39,6 +39,8 @@ import java.util.List; import java.util.function.Predicate; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.VMError; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -300,6 +302,7 @@ public String helpText() { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { + SubstrateOptions.UseParallelGC.update(values, false); SubstrateOptions.UseEpsilonGC.update(values, false); } } @@ -312,10 +315,33 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseParallelGC.update(values, false); } } }; + @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// + @Option(help = "Use a parallel GC")// + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + if (newValue) { + SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseEpsilonGC.update(values, false); + ConcealedOptions.UseRememberedSet.update(values, false); + SubstrateOptions.MultiThreaded.update(values, true); + } + } + }; + + @Fold + public static boolean useGraalCeGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); + } + + @Option(help = "Number of parallel GC worker threads.", type = OptionType.User)// + public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0L); + @Option(help = "The size of each thread stack at run-time, in bytes.", type = OptionType.User)// public static final RuntimeOptionKey StackSize = new RuntimeOptionKey<>(0L); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 05f89811bce0..f081e00bca5d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); + ParallelGC.start(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index c62182edba9f..f161bef7c9d1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,7 +1,31 @@ package com.oracle.svm.core.heap; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.log.Log; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; + public abstract class ParallelGC { - protected volatile boolean stopped; + + @Fold + public static ParallelGC singleton() { + return ImageSingletons.lookup(ParallelGC.class); + } + + @Fold + protected static boolean isSupported() { + return SubstrateOptions.UseParallelGC.getValue(); + } + + public static void start() { + Log log = Log.log().string("ParGC ");/// + if (isSupported()) { + log.string("supported").newline(); + singleton().startWorkerThreads(); + } else { + log.string("missing").newline(); + } + } public abstract void startWorkerThreads(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java index 23bb8ea222b0..7c92b153de27 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java @@ -118,7 +118,7 @@ final class PodFeature implements PodSupport, Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override From 4c77aa3e1fae306860b1d93add67fe32b9dbc6c5 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 13:11:47 +0300 Subject: [PATCH 49/99] Removed some static state from ParallelGCImpl --- .../oracle/svm/core/genscavenge/GCImpl.java | 2 - .../genscavenge/GreyToBlackObjectVisitor.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 45 ++++-- .../genscavenge/parallel/ParallelGCImpl.java | 132 ++++++++++-------- .../com/oracle/svm/core/heap/ParallelGC.java | 9 +- 5 files changed, 107 insertions(+), 87 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ee38df73b794..ce972b44606f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -165,7 +165,6 @@ private void collect(GCCause cause, boolean forceFullGC) { if (outOfMemory) { throw OutOfMemoryUtil.heapSizeExceeded(); } - ParallelGCImpl.checkThrowable(); } } @@ -1148,7 +1147,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); ///rm? } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index ccce076078a1..ac996cd2c6b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,6 +25,8 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -58,8 +60,8 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (!ParallelGCImpl.isEnabled()) { - ParallelGCImpl.push(Word.objectToUntrackedPointer(o)); + if (ParallelGC.isSupported() && !ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index bebbe655e344..5fec82ce51cb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; @@ -171,7 +172,12 @@ public Log report(Log log, boolean traceHeapChunks) { private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + AlignedHeapChunk.AlignedHeader oldChunk; + if (ParallelGC.isSupported()) { + oldChunk = ParallelGCImpl.getThreadLocalChunk(); + } else { + oldChunk = getLastAlignedHeapChunk(); + } if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -179,13 +185,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - mutex.lock(); - try { - result = allocateInNewChunk(objectSize); - return result; - } finally { - mutex.unlock(); - } + return allocateInNewChunk(objectSize); } /** @@ -193,14 +193,25 @@ private Pointer allocateMemory(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer freeMemory(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + assert ParallelGCImpl.isInParallelPhase(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); } private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - ParallelGCImpl.TLAB.set(newChunk); + AlignedHeapChunk.AlignedHeader newChunk; + if (ParallelGC.isSupported()) { + mutex.lock(); + try { + newChunk = requestAlignedHeapChunk(); + } finally { + mutex.unlock(); + } + ParallelGCImpl.setThreadLocalChunk(newChunk); + } else { + newChunk = requestAlignedHeapChunk(); + } if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } @@ -215,6 +226,10 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { lastAlignedHeapChunk = WordFactory.nullPointer(); firstUnalignedHeapChunk = WordFactory.nullPointer(); lastUnalignedHeapChunk = WordFactory.nullPointer(); + + if (ParallelGC.isSupported()) { + ParallelGCImpl.killThreadLocalChunk(); + } accounting.reset(); } @@ -371,7 +386,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -428,7 +443,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.localPush(copyMemory); + ParallelGCImpl.singleton().pushLocally(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -484,7 +499,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina mutex.unlock(); } - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } @@ -512,7 +527,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o mutex.unlock(); } - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index c89638509b14..d854952312ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,5 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; @@ -14,6 +13,7 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -25,23 +25,22 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 4; + private static final int WORKERS_COUNT = 4; /// tlab of the current to-space - public static final FastThreadLocalWord TLAB = - FastThreadLocalFactory.createWord("ParallelGCImpl.TLAB"); + private static final FastThreadLocalWord chunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); private static final VMMutex mutex = new VMMutex("pargc"); private static final VMCondition cond = new VMCondition(mutex); - private static AtomicInteger busy = new AtomicInteger(0); + private final AtomicInteger busy = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); private static final ThreadLocal localStack = new ThreadLocal<>(); - private static int currentStack; + private int currentStack; - private static volatile boolean enabled; - private static volatile Throwable throwable; + private volatile boolean inParallelPhase; @Override public void startWorkerThreads() { @@ -52,36 +51,32 @@ public void startWorkerThreads() { IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); } - public void startWorkerThread(int n) { + private void startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); final ThreadLocalTaskStack stack = STACKS[n]; - localStack.set(STACKS[n]); + localStack.set(stack); getStats().install(); - try { - while (true) { - mutex.lock(); - while (busy.get() < WORKERS_COUNT) { - log().string("WW block ").unsigned(n).newline(); - cond.block(); - } - mutex.unlock(); - - log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); - Object obj; - while ((obj = stack.pop()) != null) { - getVisitor().doVisitObject(obj); - } - TLAB.set(WordFactory.nullPointer()); - if (busy.decrementAndGet() <= 0) { - cond.broadcast(); - } - log().string("WW idle ").unsigned(n).newline(); + while (true) { + mutex.lock(); + while (busy.get() < WORKERS_COUNT) { + log().string("WW block ").unsigned(n).newline(); + cond.block(); } - } catch (Throwable e) { - throwable = e; + mutex.unlock(); + + log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); + Object obj; + while ((obj = stack.pop()) != null) { + getVisitor().doVisitObject(obj); + } + killThreadLocalChunk(); + if (busy.decrementAndGet() <= 0) { + cond.broadcast(); + } + log().string("WW idle ").unsigned(n).newline(); } }); t.setName("ParallelGCWorker-" + n); @@ -89,60 +84,75 @@ public void startWorkerThread(int n) { t.start(); } - public static GreyToBlackObjectVisitor getVisitor() { + private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } /// name, explain - public static void push(Pointer ptr) { + public void push(Pointer ptr) { + assert !isInParallelPhase(); push(ptr, STACKS[currentStack]); currentStack = (currentStack + 1) % WORKERS_COUNT; } - public static void localPush(Pointer ptr) { + public void pushLocally(Pointer ptr) { + assert isInParallelPhase(); push(ptr, localStack.get()); } - private static void push(Pointer ptr, ThreadLocalTaskStack stack) { + private void push(Pointer ptr, ThreadLocalTaskStack stack) { if (!stack.push(ptr)) { getVisitor().doVisitObject(ptr.toObject()); } } - public static void waitForIdle() { - if (isSupported()) { - setEnabled(true); - - log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run + @Fold + public static ParallelGCImpl singleton() { + return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + } - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle - } - mutex.unlock(); + public static boolean isInParallelPhase() { + return isSupported() && singleton().inParallelPhase; + } - setEnabled(false); + public static void waitForIdle() { + if (isSupported()) { + singleton().waitForIdleImpl(); } } - public static boolean isEnabled() { - return enabled; + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return chunkTL.get(); + } + + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + chunkTL.set(chunk); } - private static void setEnabled(boolean enabled) { - ParallelGCImpl.enabled = enabled; + public static void killThreadLocalChunk() { + chunkTL.set(WordFactory.nullPointer()); } - public static void checkThrowable() { - if (throwable != null) { - Log.log().string("PGC error : ").string(throwable.getClass().getName()) - .string(" : ").string(throwable.getMessage()).newline(); - throwable.printStackTrace(); - throw new Error(throwable); + private void waitForIdleImpl() { + setInParallelPhase(true); + + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle } + mutex.unlock(); + + setInParallelPhase(false); + } + + private void setInParallelPhase(boolean inParallelPhase) { + assert isSupported(); + singleton().inParallelPhase = inParallelPhase; } public static Stats getStats() { @@ -150,7 +160,7 @@ public static Stats getStats() { } static Log log() { - return Log.noopLog(); + return Log.log();/// } } @@ -159,7 +169,7 @@ class ParallelGCFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseParallelGC.getValue(); + return ParallelGC.isSupported(); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index f161bef7c9d1..cc498ea348ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -8,12 +8,7 @@ public abstract class ParallelGC { @Fold - public static ParallelGC singleton() { - return ImageSingletons.lookup(ParallelGC.class); - } - - @Fold - protected static boolean isSupported() { + public static boolean isSupported() { return SubstrateOptions.UseParallelGC.getValue(); } @@ -21,7 +16,7 @@ public static void start() { Log log = Log.log().string("ParGC ");/// if (isSupported()) { log.string("supported").newline(); - singleton().startWorkerThreads(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); } else { log.string("missing").newline(); } From 3d6b797cb8e5595559c9de9bb20a85cfe66f7cda Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 15:26:19 +0300 Subject: [PATCH 50/99] Replaced `Space.mutex` with `ParallelGCImpl.mutex` --- .../oracle/svm/core/genscavenge/Space.java | 46 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5fec82ce51cb..a77e18189428 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -62,7 +62,6 @@ public final class Space { private final boolean isFromSpace; private final int age; private final ChunksAccounting accounting; - private final VMMutex mutex; /* Heads and tails of the HeapChunk lists. */ private AlignedHeapChunk.AlignedHeader firstAlignedHeapChunk; @@ -87,7 +86,6 @@ public final class Space { this.isFromSpace = isFromSpace; this.age = age; this.accounting = new ChunksAccounting(accounting); - this.mutex = new VMMutex(name + "-mutex"); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -200,17 +198,13 @@ private Pointer freeMemory(UnsignedWord objectSize) { } private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk; if (ParallelGC.isSupported()) { - mutex.lock(); - try { - newChunk = requestAlignedHeapChunk(); - } finally { - mutex.unlock(); - } + ParallelGCImpl.mutex.lock(); + } + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + if (ParallelGC.isSupported()) { + ParallelGCImpl.mutex.unlock(); ParallelGCImpl.setThreadLocalChunk(newChunk); - } else { - newChunk = requestAlignedHeapChunk(); } if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); @@ -491,15 +485,14 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - mutex.lock(); - try { - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); - } finally { - mutex.unlock(); + boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + if (parallel) { + ParallelGCImpl.mutex.lock(); } - - if (ParallelGCImpl.isInParallelPhase()) { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + if (parallel) { + ParallelGCImpl.mutex.unlock(); if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } @@ -519,15 +512,14 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - mutex.lock(); - try { - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); - } finally { - mutex.unlock(); + boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + if (parallel) { + ParallelGCImpl.mutex.lock(); } - - if (ParallelGCImpl.isInParallelPhase()) { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + if (parallel) { + ParallelGCImpl.mutex.unlock(); if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d854952312ed..3c386e843f1b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -31,8 +31,8 @@ public class ParallelGCImpl extends ParallelGC { private static final FastThreadLocalWord chunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); - private static final VMMutex mutex = new VMMutex("pargc"); - private static final VMCondition cond = new VMCondition(mutex); + public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); + private final VMCondition cond = new VMCondition(mutex); private final AtomicInteger busy = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = From 4a3c01c616e68087f16ec4fd4f62a624ea15de2c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 16:53:15 +0300 Subject: [PATCH 51/99] Cleanup: removed Stats printout --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 +++++--- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 14 +++++++------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ce972b44606f..4c04753fdb73 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -129,6 +129,8 @@ public final class GCImpl implements GC { public String getName() { if (SubstrateOptions.UseEpsilonGC.getValue()) { return "Epsilon GC"; + } else if (ParallelGC.isSupported()) { + return "Parallel GC"; } else { return "Serial GC"; } @@ -206,7 +208,6 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); - ParallelGCImpl.getStats().reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -434,7 +435,6 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - ParallelGCImpl.getStats().print(Log.log()); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } @@ -1051,7 +1051,9 @@ private void scanGreyObjects(boolean isIncremental) { scanGreyObjectsLoop(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - ParallelGCImpl.waitForIdle(); + if (ParallelGC.isSupported()) { + ParallelGCImpl.waitForIdle(); + } } } finally { scanGreyObjectsTimer.close(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index a77e18189428..1b353f2bed7b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -380,7 +380,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGC.isSupported() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3c386e843f1b..0ad3f33a0c08 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -24,10 +24,12 @@ public class ParallelGCImpl extends ParallelGC { - /// static -> ImageSingletons + /// determine at runtime? private static final int WORKERS_COUNT = 4; - /// tlab of the current to-space + /** + * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. + */ private static final FastThreadLocalWord chunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); @@ -44,7 +46,6 @@ public class ParallelGCImpl extends ParallelGC { @Override public void startWorkerThreads() { - /// determine worker count int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); @@ -112,13 +113,12 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; + assert isSupported(); + return singleton().inParallelPhase; } public static void waitForIdle() { - if (isSupported()) { - singleton().waitForIdleImpl(); - } + singleton().waitForIdleImpl(); } public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { From d99ca0f666144e7796034f1d08ad5a63d6de05a2 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 10:53:05 +0300 Subject: [PATCH 52/99] Enqueue objects as they are copied `scanGreyObjects` is a noop --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 8 +++----- .../src/com/oracle/svm/core/genscavenge/Space.java | 3 +++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 4c04753fdb73..4ab75b898420 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1043,17 +1043,15 @@ private static void prepareForPromotion(boolean isIncremental) { } } - /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (isIncremental) { + if (ParallelGC.isSupported()) { + ParallelGCImpl.waitForIdle(); + } else if (isIncremental) { scanGreyObjectsLoop(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - if (ParallelGC.isSupported()) { - ParallelGCImpl.waitForIdle(); - } } } finally { scanGreyObjectsTimer.close(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 1b353f2bed7b..182ca48f9a62 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -387,6 +387,9 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { ObjectHeaderImpl.installForwardingPointer(original, copy); + if (ParallelGC.isSupported()) { + ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(copy)); + } } return copy; } From e29e81debb622fdf663c35ac0df2ad5a13169c6e Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 11:13:28 +0300 Subject: [PATCH 53/99] Cleanup --- .../oracle/svm/core/genscavenge/GCImpl.java | 8 +++---- .../genscavenge/GreyToBlackObjectVisitor.java | 5 ++--- .../oracle/svm/core/genscavenge/Space.java | 21 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 15 ++++++++----- .../com/oracle/svm/core/SubstrateOptions.java | 5 ----- .../graal/snippets/CEntryPointSnippets.java | 4 ++-- .../com/oracle/svm/core/heap/ParallelGC.java | 11 +++------- 7 files changed, 30 insertions(+), 39 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 4ab75b898420..b66b85a79047 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,9 +29,6 @@ import java.lang.ref.Reference; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; @@ -69,12 +66,15 @@ import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.CodeReferenceMapDecoder; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; @@ -129,7 +129,7 @@ public final class GCImpl implements GC { public String getName() { if (SubstrateOptions.UseEpsilonGC.getValue()) { return "Epsilon GC"; - } else if (ParallelGC.isSupported()) { + } else if (SubstrateOptions.UseParallelGC.getValue()) { return "Parallel GC"; } else { return "Serial GC"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index ac996cd2c6b5..0f725cc1cb92 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,15 +24,14 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 182ca48f9a62..f8f34b2427ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,10 +28,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.annotate.AlwaysInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -44,12 +40,15 @@ import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; /** * A Space is a collection of HeapChunks. @@ -380,7 +379,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGC.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -440,7 +439,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushLocally(copyMemory); + ParallelGCImpl.singleton().pushToLocalStack(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -488,13 +487,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); @@ -515,13 +513,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0ad3f33a0c08..8c95df1f9cef 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -45,7 +45,7 @@ public class ParallelGCImpl extends ParallelGC { private volatile boolean inParallelPhase; @Override - public void startWorkerThreads() { + public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); @@ -89,14 +89,20 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - /// name, explain + /** + * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' stacks + * in round-robin fashion, attempting to balance load between workers. + */ public void push(Pointer ptr) { assert !isInParallelPhase(); push(ptr, STACKS[currentStack]); currentStack = (currentStack + 1) % WORKERS_COUNT; } - public void pushLocally(Pointer ptr) { + /** + * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local stack. + */ + public void pushToLocalStack(Pointer ptr) { assert isInParallelPhase(); push(ptr, localStack.get()); } @@ -113,8 +119,7 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - assert isSupported(); - return singleton().inParallelPhase; + return isSupported() && singleton().inParallelPhase; } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 94ff01a0fd98..5fa114886186 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -39,8 +39,6 @@ import java.util.List; import java.util.function.Predicate; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.util.VMError; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -339,9 +337,6 @@ public static boolean useGraalCeGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } - @Option(help = "Number of parallel GC worker threads.", type = OptionType.User)// - public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0L); - @Option(help = "The size of each thread stack at run-time, in bytes.", type = OptionType.User)// public static final RuntimeOptionKey StackSize = new RuntimeOptionKey<>(0L); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index f081e00bca5d..14f67dd1a2c7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -33,7 +33,6 @@ import java.util.Map; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -87,6 +86,7 @@ import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode; import com.oracle.svm.core.graal.nodes.CEntryPointLeaveNode; import com.oracle.svm.core.graal.nodes.CEntryPointUtilityNode; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ParallelGC.start(); + ParallelGC.startWorkerThreads(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index cc498ea348ca..3bd3cf42b34a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,7 +1,6 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; @@ -12,15 +11,11 @@ public static boolean isSupported() { return SubstrateOptions.UseParallelGC.getValue(); } - public static void start() { - Log log = Log.log().string("ParGC ");/// + public static void startWorkerThreads() { if (isSupported()) { - log.string("supported").newline(); - ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); - } else { - log.string("missing").newline(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); } } - public abstract void startWorkerThreads(); + public abstract void startWorkerThreadsImpl(); } From e068f43258dc74012ecfe2e5caa9d49fb7370a42 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 16:10:00 +0300 Subject: [PATCH 54/99] Fixed hangup; better worker thread management --- .../genscavenge/parallel/ParallelGCImpl.java | 57 ++++++++++--------- .../parallel/ThreadLocalTaskStack.java | 2 +- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8c95df1f9cef..e1b8129bfbf4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -24,8 +24,10 @@ public class ParallelGCImpl extends ParallelGC { - /// determine at runtime? + /// determine at runtime, max 32 (from busyWorkers) private static final int WORKERS_COUNT = 4; + private static final int WORKERS_ALL = 0b1111; + private static final int WORKERS_NONE = 0; /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -34,8 +36,9 @@ public class ParallelGCImpl extends ParallelGC { FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); - private final VMCondition cond = new VMCondition(mutex); - private final AtomicInteger busy = new AtomicInteger(0); + private final VMCondition seqPhase = new VMCondition(mutex); + private final VMCondition parPhase = new VMCondition(mutex); + private final AtomicInteger busyWorkers = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); @@ -61,12 +64,12 @@ private void startWorkerThread(int n) { localStack.set(stack); getStats().install(); while (true) { - mutex.lock(); - while (busy.get() < WORKERS_COUNT) { + do { log().string("WW block ").unsigned(n).newline(); - cond.block(); - } - mutex.unlock(); + mutex.lock(); + parPhase.block(); + mutex.unlock(); + } while ((busyWorkers.get() & (1 << n)) == 0); log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); Object obj; @@ -74,8 +77,15 @@ private void startWorkerThread(int n) { getVisitor().doVisitObject(obj); } killThreadLocalChunk(); - if (busy.decrementAndGet() <= 0) { - cond.broadcast(); + + int witness = busyWorkers.get(); + int expected; + do { + expected = witness; + witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); + } while (witness != expected); + if (busyWorkers.get() == WORKERS_NONE) { + seqPhase.signal(); } log().string("WW idle ").unsigned(n).newline(); } @@ -139,25 +149,20 @@ public static void killThreadLocalChunk() { } private void waitForIdleImpl() { - setInParallelPhase(true); + inParallelPhase = true; log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run - - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle + busyWorkers.set(WORKERS_ALL); + parPhase.broadcast(); // let worker threads run + + while (busyWorkers.get() != WORKERS_NONE) { + mutex.lock(); + log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + seqPhase.block(); // wait for them to become idle + mutex.unlock(); } - mutex.unlock(); - - setInParallelPhase(false); - } - private void setInParallelPhase(boolean inParallelPhase) { - assert isSupported(); - singleton().inParallelPhase = inParallelPhase; + inParallelPhase = false; } public static Stats getStats() { @@ -165,7 +170,7 @@ public static Stats getStats() { } static Log log() { - return Log.log();/// + return Log.noopLog();/// } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java index 09645ebcc404..eb1201dccf89 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java @@ -33,5 +33,5 @@ Object pop() { int size() { return top; - } + } ///rm? } \ No newline at end of file From e67370c9724364fa38f6f8852f1d4c2b26c75e74 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 17:10:54 +0300 Subject: [PATCH 55/99] Reuse GC thread as one of the workers --- .../oracle/svm/core/genscavenge/Space.java | 6 +- .../core/genscavenge/parallel/CasQueue.java | 85 ----------- .../genscavenge/parallel/ParallelGCImpl.java | 140 +++++++++--------- .../core/genscavenge/parallel/TaskQueue.java | 105 ------------- ...lTaskStack.java => ThreadLocalBuffer.java} | 6 +- 5 files changed, 76 insertions(+), 266 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/{ThreadLocalTaskStack.java => ThreadLocalBuffer.java} (85%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index f8f34b2427ed..8b8ca33cb467 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -219,10 +219,6 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { lastAlignedHeapChunk = WordFactory.nullPointer(); firstUnalignedHeapChunk = WordFactory.nullPointer(); lastUnalignedHeapChunk = WordFactory.nullPointer(); - - if (ParallelGC.isSupported()) { - ParallelGCImpl.killThreadLocalChunk(); - } accounting.reset(); } @@ -439,7 +435,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushToLocalStack(copyMemory); + ParallelGCImpl.singleton().pushToLocalBuffer(copyMemory); return copy; } else { // Retract speculatively allocated memory diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java deleted file mode 100644 index 107fedc3afbd..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; - -import java.util.concurrent.atomic.AtomicInteger; - -/// Lame, test only. Doesn't handle overflows -public class CasQueue { - private static final int SIZE = 1024 * 1024; - - final Stats stats = new Stats(); - - private final Pointer[] data = new Pointer[SIZE]; - private final AtomicInteger getIndex = new AtomicInteger(0); - private final AtomicInteger putIndex = new AtomicInteger(0); - - private Log log() { - return ParallelGCImpl.log(); - } - - public boolean put(Pointer ptr) { - int cur = putIndex.get(); - int witness; - while ((witness = putIndex.compareAndExchange(cur, cur + 1)) != cur) { - cur = witness; - } - data[cur] = ptr; - return true; - } - - public boolean consume(Consumer consumer) { - int cur = getIndex.get(); - int witness; - while (cur < putIndex.get() && (witness = getIndex.compareAndExchange(cur, cur + 1)) != cur) { - cur = witness; - } - if (cur >= putIndex.get()) { - return false; - } - Object obj = data[cur].toObject(); - consumer.accept(obj); - return true; - } - - public void drain(Consumer consumer) { - while (consume(consumer)); - getIndex.set(0); - putIndex.set(0); - } - - interface Consumer { - void accept(Object object); - } - - // Non MT safe, needs locking - public static class Stats { - private int count, maxSize; - - void noteTask(int putIndex, int getIndex) { - int size = putIndex - getIndex; - if (size < 0) { - size += SIZE; - } - if (size > maxSize) { - maxSize = size; - } - count++; - } - - public int getCount() { - return count; - } - - public int getMaxSize() { - return maxSize; - } - - public void reset() { - count = maxSize = 0; - } - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e1b8129bfbf4..79ad941a9c9a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -11,6 +11,7 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; @@ -26,9 +27,15 @@ public class ParallelGCImpl extends ParallelGC { /// determine at runtime, max 32 (from busyWorkers) private static final int WORKERS_COUNT = 4; - private static final int WORKERS_ALL = 0b1111; + private static final int WORKERS_ALL = 0b1110; private static final int WORKERS_NONE = 0; + private static final ThreadLocalBuffer[] BUFFERS = + IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); + private static final FastThreadLocalObject localBuffer = + FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); + private int currentBuffer; + /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -40,19 +47,59 @@ public class ParallelGCImpl extends ParallelGC { private final VMCondition parPhase = new VMCondition(mutex); private final AtomicInteger busyWorkers = new AtomicInteger(0); - private static final ThreadLocalTaskStack[] STACKS = - IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); - private static final ThreadLocal localStack = new ThreadLocal<>(); - private int currentStack; - private volatile boolean inParallelPhase; + @Fold + public static ParallelGCImpl singleton() { + return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + } + + public static boolean isInParallelPhase() { + return isSupported() && singleton().inParallelPhase; + } + + public static void waitForIdle() { + singleton().waitForIdleImpl(); + } + + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return chunkTL.get(); + } + + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + chunkTL.set(chunk); + } + + /** + * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' buffers + * in round-robin fashion, attempting to balance load between workers. + */ + public void push(Pointer ptr) { + assert !isInParallelPhase(); + push(ptr, BUFFERS[currentBuffer]); + currentBuffer = (currentBuffer + 1) % WORKERS_COUNT; + } + + /** + * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local buffer. + */ + public void pushToLocalBuffer(Pointer ptr) { + assert isInParallelPhase(); + push(ptr, localBuffer.get()); + } + + private void push(Pointer ptr, ThreadLocalBuffer buffer) { + if (!buffer.push(ptr)) { + getVisitor().doVisitObject(ptr.toObject()); + } + } + @Override public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); + IntStream.range(1, WORKERS_COUNT).forEach(this::startWorkerThread); } private void startWorkerThread(int n) { @@ -60,8 +107,8 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); - final ThreadLocalTaskStack stack = STACKS[n]; - localStack.set(stack); + final ThreadLocalBuffer buffer = BUFFERS[n]; + localBuffer.set(buffer); getStats().install(); while (true) { do { @@ -71,13 +118,6 @@ private void startWorkerThread(int n) { mutex.unlock(); } while ((busyWorkers.get() & (1 << n)) == 0); - log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); - Object obj; - while ((obj = stack.pop()) != null) { - getVisitor().doVisitObject(obj); - } - killThreadLocalChunk(); - int witness = busyWorkers.get(); int expected; do { @@ -95,59 +135,6 @@ private void startWorkerThread(int n) { t.start(); } - private GreyToBlackObjectVisitor getVisitor() { - return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); - } - - /** - * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' stacks - * in round-robin fashion, attempting to balance load between workers. - */ - public void push(Pointer ptr) { - assert !isInParallelPhase(); - push(ptr, STACKS[currentStack]); - currentStack = (currentStack + 1) % WORKERS_COUNT; - } - - /** - * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local stack. - */ - public void pushToLocalStack(Pointer ptr) { - assert isInParallelPhase(); - push(ptr, localStack.get()); - } - - private void push(Pointer ptr, ThreadLocalTaskStack stack) { - if (!stack.push(ptr)) { - getVisitor().doVisitObject(ptr.toObject()); - } - } - - @Fold - public static ParallelGCImpl singleton() { - return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); - } - - public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; - } - - public static void waitForIdle() { - singleton().waitForIdleImpl(); - } - - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { - return chunkTL.get(); - } - - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { - chunkTL.set(chunk); - } - - public static void killThreadLocalChunk() { - chunkTL.set(WordFactory.nullPointer()); - } - private void waitForIdleImpl() { inParallelPhase = true; @@ -155,6 +142,9 @@ private void waitForIdleImpl() { busyWorkers.set(WORKERS_ALL); parPhase.broadcast(); // let worker threads run + localBuffer.set(BUFFERS[0]); + drain(BUFFERS[0]); + while (busyWorkers.get() != WORKERS_NONE) { mutex.lock(); log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); @@ -165,6 +155,18 @@ private void waitForIdleImpl() { inParallelPhase = false; } + private void drain(ThreadLocalBuffer buffer) { + Object obj; + while ((obj = buffer.pop()) != null) { + getVisitor().doVisitObject(obj); + } + chunkTL.set(WordFactory.nullPointer()); + } + + private GreyToBlackObjectVisitor getVisitor() { + return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); + } + public static Stats getStats() { return Stats.stats(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java deleted file mode 100644 index aaafd18fdea6..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; - -public class TaskQueue { - private static final int SIZE = 128 * 1024; - - private final VMMutex mutex; - private final VMCondition cond; - private final Pointer[] data; - private int idleCount; - private int getIndex; - private int putIndex; - - public TaskQueue(String name) { - mutex = new VMMutex(name + "-queue"); - cond = new VMCondition(mutex); - data = new Pointer[SIZE]; - } - - private Log log() { - return Log.noopLog(); - } - - private int next(int index) { - return (index + 1) % SIZE; - } - - private boolean canGet() { - return getIndex != putIndex; - } - - private boolean canPut() { - return next(putIndex) != getIndex; - } - - public boolean put(Pointer ptr) { - try { - mutex.lock(); - if (!canPut()) { - log().string("TQ cannot put task\n"); - return false; - } - data[putIndex] = ptr; - putIndex = next(putIndex); - } finally { - mutex.unlock(); - } - cond.broadcast(); - return true; - } - - public void consume(Consumer consumer) { - Pointer ptr; - mutex.lock(); - try { - while (!canGet()) { - log().string("TQ cannot get task\n"); - idleCount++; - cond.block(); - idleCount--; - } - ptr = data[getIndex]; - getIndex = next(getIndex); - } finally { - mutex.unlock(); - } - cond.broadcast(); - consumer.accept(ptr.toObject()); - } - - // Non MT safe. Only call when no workers are running - public void drain(Consumer consumer) { - while (canGet()) { - Object obj = data[getIndex].toObject(); - consumer.accept(obj); - getIndex = next(getIndex); - } - } - - public void waitUntilIdle(int expectedIdleCount) { - log().string("TQ waitForIdle\n"); - while (true) { - try { - mutex.lock(); - while (canGet()) { - cond.block(); - } - if (idleCount >= expectedIdleCount) { - log().string("TQ waitForIdle over\n"); - return; - } - } finally { - mutex.unlock(); - } - } - } - - interface Consumer { - void accept(Object object); - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java similarity index 85% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java index eb1201dccf89..c34af0a589df 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java @@ -3,8 +3,10 @@ import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; -/// thread local, non MT safe -public class ThreadLocalTaskStack { +/** + * Thread local (non MT safe) buffer where pointers to grey objects are stored. + */ +public class ThreadLocalBuffer { private static final int SIZE = 128 * 1024; private final Pointer[] data = new Pointer[SIZE]; From 8e33c3d0e752ffe7aa25ae7b404876852481a8b0 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 26 Aug 2022 10:06:59 +0300 Subject: [PATCH 56/99] Determine number of workers at runtime --- .../oracle/svm/core/genscavenge/Space.java | 7 +- .../genscavenge/parallel/ParallelGCImpl.java | 111 ++++++++++++------ .../parallel/ParallelGCOptions.java | 11 ++ .../parallel/ThreadLocalBuffer.java | 54 ++++++--- .../graal/snippets/CEntryPointSnippets.java | 5 +- 5 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8b8ca33cb467..d81a25b28bc5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -394,7 +394,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); Pointer originalMemory = Word.objectToUntrackedPointer(original); UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { @@ -403,7 +402,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. // If another thread copies the object first, we retract the allocation later. - UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); + UnsignedWord size = getSizeFromHeader(original, originalHeader); assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { @@ -445,8 +444,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } @AlwaysInline("GC performance") - public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header, ObjectHeaderImpl impl) { - int encoding = impl.dynamicHubFromObjectHeader(header).getLayoutEncoding(); + public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header) { + int encoding = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header).getLayoutEncoding(); return LayoutEncoding.getSizeFromEncoding(obj, encoding); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 79ad941a9c9a..d99f6b454d38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,11 +1,13 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; @@ -25,13 +27,19 @@ public class ParallelGCImpl extends ParallelGC { - /// determine at runtime, max 32 (from busyWorkers) - private static final int WORKERS_COUNT = 4; - private static final int WORKERS_ALL = 0b1110; - private static final int WORKERS_NONE = 0; + private static final int MAX_WORKERS_COUNT = 31; + /** + * Number of parallel GC workers. Default is 1 so that if GC happens before worker threads are started, + * it occurs on a single thread just like serial GC. + */ + private int workerCount = 1; + private int allWorkersBusy; + private final AtomicInteger busyWorkers = new AtomicInteger(0); + + /// create only as many as needed private static final ThreadLocalBuffer[] BUFFERS = - IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); + IntStream.range(0, MAX_WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); private static final FastThreadLocalObject localBuffer = FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); private int currentBuffer; @@ -45,7 +53,6 @@ public class ParallelGCImpl extends ParallelGC { public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); - private final AtomicInteger busyWorkers = new AtomicInteger(0); private volatile boolean inParallelPhase; @@ -76,8 +83,8 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { */ public void push(Pointer ptr) { assert !isInParallelPhase(); - push(ptr, BUFFERS[currentBuffer]); - currentBuffer = (currentBuffer + 1) % WORKERS_COUNT; + BUFFERS[currentBuffer].push(ptr); + currentBuffer = (currentBuffer + 1) % workerCount; } /** @@ -85,13 +92,7 @@ public void push(Pointer ptr) { */ public void pushToLocalBuffer(Pointer ptr) { assert isInParallelPhase(); - push(ptr, localBuffer.get()); - } - - private void push(Pointer ptr, ThreadLocalBuffer buffer) { - if (!buffer.push(ptr)) { - getVisitor().doVisitObject(ptr.toObject()); - } + localBuffer.get().push(ptr); } @Override @@ -99,35 +100,63 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - IntStream.range(1, WORKERS_COUNT).forEach(this::startWorkerThread); + workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker + allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); + for (int i = 0; i < workerCount; i++) { + BUFFERS[i].runtimeInit(); + if (i > 0) { + // We reuse the gc thread for one of the workers, so number of worker threads is `workerCount - 1` + startWorkerThread(i); + } + } + } + + private int getWorkerCount() { + int setting = ParallelGCOptions.ParallelGCWorkers.getValue(); + int workers = setting > 0 ? setting : getDefaultWorkerCount(); + workers = Math.min(workers, MAX_WORKERS_COUNT); + verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); + return workers; + } + + private int getDefaultWorkerCount() { + // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() + int cpus = Jvm.JVM_ActiveProcessorCount(); + return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } private void startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); - log().string("WW start ").unsigned(n).newline(); + debugLog().string("WW start ").unsigned(n).newline(); final ThreadLocalBuffer buffer = BUFFERS[n]; localBuffer.set(buffer); getStats().install(); while (true) { - do { - log().string("WW block ").unsigned(n).newline(); - mutex.lock(); - parPhase.block(); - mutex.unlock(); - } while ((busyWorkers.get() & (1 << n)) == 0); - - int witness = busyWorkers.get(); - int expected; - do { - expected = witness; - witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); - } while (witness != expected); - if (busyWorkers.get() == WORKERS_NONE) { - seqPhase.signal(); + try { + do { + debugLog().string("WW block ").unsigned(n).newline(); + mutex.lock(); + parPhase.block(); + mutex.unlock(); + } while ((busyWorkers.get() & (1 << n)) == 0); + + debugLog().string("WW run ").unsigned(n).newline(); + drain(buffer); + int witness = busyWorkers.get(); + int expected; + do { + expected = witness; + witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); + } while (witness != expected); + if (busyWorkers.get() == 0) { + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); } - log().string("WW idle ").unsigned(n).newline(); } }); t.setName("ParallelGCWorker-" + n); @@ -138,16 +167,16 @@ private void startWorkerThread(int n) { private void waitForIdleImpl() { inParallelPhase = true; - log().string("PP start workers\n"); - busyWorkers.set(WORKERS_ALL); + debugLog().string("PP start workers\n"); + busyWorkers.set(allWorkersBusy); parPhase.broadcast(); // let worker threads run localBuffer.set(BUFFERS[0]); drain(BUFFERS[0]); - while (busyWorkers.get() != WORKERS_NONE) { + while (busyWorkers.get() != 0) { mutex.lock(); - log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); seqPhase.block(); // wait for them to become idle mutex.unlock(); } @@ -171,8 +200,12 @@ public static Stats getStats() { return Stats.stats(); } - static Log log() { - return Log.noopLog();/// + private static Log verboseGCLog() { + return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); + } + + static Log debugLog() {///rm + return Log.noopLog(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java new file mode 100644 index 000000000000..208f06fc633e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -0,0 +1,11 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.option.RuntimeOptionKey; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionType; + +public class ParallelGCOptions { + + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) + public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0); +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java index c34af0a589df..1dc025567def 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java @@ -1,39 +1,61 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.log.Log; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; /** * Thread local (non MT safe) buffer where pointers to grey objects are stored. */ public class ThreadLocalBuffer { - private static final int SIZE = 128 * 1024; + private static final int INITIAL_SIZE = 128 * 1024 * 8; // 128k words + private static final OutOfMemoryError OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate a ThreadLocalBuffer"); - private final Pointer[] data = new Pointer[SIZE]; - private int top = 0; + private Pointer buffer = WordFactory.nullPointer(); + private UnsignedWord size = WordFactory.unsigned(INITIAL_SIZE); + private UnsignedWord top = WordFactory.zero(); private Log log() { - return ParallelGCImpl.log(); + return Log.log(); } - boolean push(Pointer ptr) { - if (top >= SIZE) { - log().string("TT cannot put task\n"); - return false; + void runtimeInit() { + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + ensureBuffer(); + } + + private void ensureCapacity() { + if (top.aboveOrEqual(size)) { + size = size.multiply(2); + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, size); + ensureBuffer(); + log().string("TLB grow to ").unsigned(size).newline(); + } + } + + private void ensureBuffer() { + if (buffer.isNull()) { + throw OutOfMemoryUtil.reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - data[top++] = ptr; - return true; + } + + void push(Pointer ptr) { + ensureCapacity(); + buffer.writeWord(top, ptr); + top = top.add(8); } Object pop() { - if (top > 0) { - return data[--top].toObjectNonNull(); + if (top.aboveThan(0)) { + top = top.subtract(8); + Pointer ptr = buffer.readWord(top); + return ptr.toObjectNonNull(); } else { return null; } } - - int size() { - return top; - } ///rm? } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 14f67dd1a2c7..6be823f70e86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,8 +292,6 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ParallelGC.startWorkerThreads(); - /* * After starting all the necessary threads, we can finally execute complex JDK code or code * that allocates a significant amount of memory. @@ -325,6 +323,9 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } + /* Number of workers can be affected by a runtime option, so start threads after options are parsed */ + ParallelGC.startWorkerThreads(); + boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now state = success ? FirstIsolateInitStates.SUCCESSFUL : FirstIsolateInitStates.FAILED; From 73177bd86ed7cf9ae4713ebf983d73a19f4a6139 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 15 Sep 2022 15:18:52 +0300 Subject: [PATCH 57/99] Shared synchronized buffer, so far holds object pointers --- .../genscavenge/parallel/ChunkBuffer.java | 37 ++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 38 +++++++------------ 2 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java new file mode 100644 index 000000000000..1da361a3609e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -0,0 +1,37 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +/** + * Synchronized buffer where chunks to be scanned are stored. + */ +public class ChunkBuffer { + private static final int SIZE = 1024 * 1024; ///handle overflow + reduce for chunks + private static final Pointer[] buffer = new Pointer[SIZE]; + private int top; + + private Log debugLog() { + return ParallelGCImpl.debugLog(); + } + + void push(Pointer ptr) { + ParallelGCImpl.mutex.lock(); + assert top < SIZE; + buffer[top++] = ptr; + ParallelGCImpl.mutex.unlock(); + } + + Object pop() { + ParallelGCImpl.mutex.lock(); + try { + if (top > 0) { + return buffer[--top].toObjectNonNull(); + } else { + return null; + } + } finally { + ParallelGCImpl.mutex.unlock(); + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d99f6b454d38..1d3148505bb4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -13,7 +13,6 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; @@ -23,7 +22,6 @@ import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -37,12 +35,7 @@ public class ParallelGCImpl extends ParallelGC { private int allWorkersBusy; private final AtomicInteger busyWorkers = new AtomicInteger(0); - /// create only as many as needed - private static final ThreadLocalBuffer[] BUFFERS = - IntStream.range(0, MAX_WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); - private static final FastThreadLocalObject localBuffer = - FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); - private int currentBuffer; + private static final ChunkBuffer buffer = new ChunkBuffer(); /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -83,8 +76,7 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { */ public void push(Pointer ptr) { assert !isInParallelPhase(); - BUFFERS[currentBuffer].push(ptr); - currentBuffer = (currentBuffer + 1) % workerCount; + buffer.push(ptr); } /** @@ -92,7 +84,7 @@ public void push(Pointer ptr) { */ public void pushToLocalBuffer(Pointer ptr) { assert isInParallelPhase(); - localBuffer.get().push(ptr); + buffer.push(ptr); } @Override @@ -102,12 +94,9 @@ public void startWorkerThreadsImpl() { workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); - for (int i = 0; i < workerCount; i++) { - BUFFERS[i].runtimeInit(); - if (i > 0) { - // We reuse the gc thread for one of the workers, so number of worker threads is `workerCount - 1` - startWorkerThread(i); - } + for (int i = 1; i < workerCount; i++) { + // We reuse the gc thread for worker #0 + startWorkerThread(i); } } @@ -130,8 +119,6 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); - final ThreadLocalBuffer buffer = BUFFERS[n]; - localBuffer.set(buffer); getStats().install(); while (true) { try { @@ -143,7 +130,7 @@ private void startWorkerThread(int n) { } while ((busyWorkers.get() & (1 << n)) == 0); debugLog().string("WW run ").unsigned(n).newline(); - drain(buffer); + drainBuffer(); int witness = busyWorkers.get(); int expected; do { @@ -151,7 +138,9 @@ private void startWorkerThread(int n) { witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); } while (witness != expected); if (busyWorkers.get() == 0) { + mutex.lock(); seqPhase.signal(); + mutex.unlock(); } debugLog().string("WW idle ").unsigned(n).newline(); } catch (Throwable ex) { @@ -171,20 +160,19 @@ private void waitForIdleImpl() { busyWorkers.set(allWorkersBusy); parPhase.broadcast(); // let worker threads run - localBuffer.set(BUFFERS[0]); - drain(BUFFERS[0]); + drainBuffer(); + mutex.lock(); while (busyWorkers.get() != 0) { - mutex.lock(); debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); seqPhase.block(); // wait for them to become idle - mutex.unlock(); } + mutex.unlock(); inParallelPhase = false; } - private void drain(ThreadLocalBuffer buffer) { + private void drainBuffer() { Object obj; while ((obj = buffer.pop()) != null) { getVisitor().doVisitObject(obj); From 060bbd47366a8c19054ffcb4e611e5bd5833d266 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 16 Sep 2022 12:41:42 +0300 Subject: [PATCH 58/99] Use buffer for aligned chunks instead of objects --- .../core/genscavenge/AlignedHeapChunk.java | 2 +- .../genscavenge/GreyToBlackObjectVisitor.java | 10 ---- .../oracle/svm/core/genscavenge/Space.java | 19 +++---- .../genscavenge/parallel/ChunkBuffer.java | 24 ++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 53 +++++++++++-------- 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 25f25d0a653d..09ccc58873d8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -147,7 +147,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") - static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { + public static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 0f725cc1cb92..870494eb4f1a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -59,16 +59,6 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (ParallelGC.isSupported() && !ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(o)); - return true; - } else { - return doVisitObject(o); - } - } - - @AlwaysInline("GC performance") - public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index d81a25b28bc5..2021a340ad57 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -182,7 +182,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(objectSize); + return allocateInNewChunk(oldChunk, objectSize); } /** @@ -196,9 +196,12 @@ private Pointer freeMemory(UnsignedWord objectSize) { return AlignedHeapChunk.freeMemory(oldChunk, objectSize); } - private Pointer allocateInNewChunk(UnsignedWord objectSize) { + private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { if (ParallelGC.isSupported()) { ParallelGCImpl.mutex.lock(); + if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { + ParallelGCImpl.singleton().push(oldChunk); + } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (ParallelGC.isSupported()) { @@ -382,9 +385,6 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { ObjectHeaderImpl.installForwardingPointer(original, copy); - if (ParallelGC.isSupported()) { - ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(copy)); - } } return copy; } @@ -434,7 +434,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushToLocalBuffer(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -488,10 +487,8 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.singleton().push(chunk); ParallelGCImpl.mutex.unlock(); - if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { - throw VMError.shouldNotReachHere(); - } } if (this.isOldSpace()) { @@ -514,10 +511,8 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { + assert false;/// ParallelGCImpl.mutex.unlock(); - if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { - throw VMError.shouldNotReachHere(); - } } if (this.isOldSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 1da361a3609e..eccd42aebc23 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,37 +1,47 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; /** * Synchronized buffer where chunks to be scanned are stored. */ public class ChunkBuffer { - private static final int SIZE = 1024 * 1024; ///handle overflow + reduce for chunks - private static final Pointer[] buffer = new Pointer[SIZE]; + private static final int SIZE = 10 * 1024; ///handle overflow + private static final HeapChunk.Header[] buffer = new HeapChunk.Header[SIZE]; private int top; private Log debugLog() { return ParallelGCImpl.debugLog(); } - void push(Pointer ptr) { + void push(HeapChunk.Header ptr) { ParallelGCImpl.mutex.lock(); assert top < SIZE; buffer[top++] = ptr; ParallelGCImpl.mutex.unlock(); } - Object pop() { + HeapChunk.Header pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - return buffer[--top].toObjectNonNull(); + return buffer[--top]; } else { - return null; + return WordFactory.nullPointer(); } } finally { ParallelGCImpl.mutex.unlock(); } } + + int size() { + ParallelGCImpl.mutex.lock(); + try { + return top; + } finally { + ParallelGCImpl.mutex.unlock(); + } + } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1d3148505bb4..b2c92634b1ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,6 +6,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; @@ -19,6 +20,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; @@ -40,8 +42,10 @@ public class ParallelGCImpl extends ParallelGC { /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ - private static final FastThreadLocalWord chunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); + private static final FastThreadLocalWord allocChunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); + private static final FastThreadLocalWord scannedChunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -62,28 +66,19 @@ public static void waitForIdle() { singleton().waitForIdleImpl(); } - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { - return chunkTL.get(); + public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { + return scannedChunkTL.get(); } - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { - chunkTL.set(chunk); + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return allocChunkTL.get(); } - /** - * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' buffers - * in round-robin fashion, attempting to balance load between workers. - */ - public void push(Pointer ptr) { - assert !isInParallelPhase(); - buffer.push(ptr); + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + allocChunkTL.set(chunk); } - /** - * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local buffer. - */ - public void pushToLocalBuffer(Pointer ptr) { - assert isInParallelPhase(); + public void push(HeapChunk.Header ptr) { buffer.push(ptr); } @@ -173,11 +168,23 @@ private void waitForIdleImpl() { } private void drainBuffer() { - Object obj; - while ((obj = buffer.pop()) != null) { - getVisitor().doVisitObject(obj); - } - chunkTL.set(WordFactory.nullPointer()); + debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); + do { + HeapChunk.Header chunk; + while ((chunk = buffer.pop()).notEqual(WordFactory.nullPointer())) { + debugLog().string("WW drain chunk=").zhex(chunk).newline(); + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) chunk, getVisitor()); + } + AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); + debugLog().string("WW drain tlab=").zhex(tlab).newline(); + if (tlab.notEqual(WordFactory.nullPointer())) { + scannedChunkTL.set(tlab); + AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); + } + } while (buffer.size() > 0 || allocChunkTL.get().notEqual(scannedChunkTL.get())); + + allocChunkTL.set(WordFactory.nullPointer()); + scannedChunkTL.set(WordFactory.nullPointer()); } private GreyToBlackObjectVisitor getVisitor() { From f0a26b72863b505cd66838c502b0aefe4568d880 Mon Sep 17 00:00:00 2001 From: peterz Date: Sat, 17 Sep 2022 16:34:07 +0300 Subject: [PATCH 59/99] Support for unaligned chunks --- .../com/oracle/svm/core/genscavenge/Space.java | 6 +++--- .../core/genscavenge/parallel/ChunkBuffer.java | 12 +++++++----- .../genscavenge/parallel/ParallelGCImpl.java | 16 +++++++++++----- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 2021a340ad57..0f265f52c7f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -200,7 +200,7 @@ private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, Unsi if (ParallelGC.isSupported()) { ParallelGCImpl.mutex.lock(); if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { - ParallelGCImpl.singleton().push(oldChunk); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); @@ -487,7 +487,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(chunk); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); ParallelGCImpl.mutex.unlock(); } @@ -511,7 +511,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - assert false;/// + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01)); ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index eccd42aebc23..69e6779d514e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -2,6 +2,7 @@ import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; /** @@ -9,21 +10,22 @@ */ public class ChunkBuffer { private static final int SIZE = 10 * 1024; ///handle overflow - private static final HeapChunk.Header[] buffer = new HeapChunk.Header[SIZE]; + private static final Pointer[] buffer = new Pointer[SIZE]; private int top; private Log debugLog() { return ParallelGCImpl.debugLog(); } - void push(HeapChunk.Header ptr) { - ParallelGCImpl.mutex.lock(); + /** + * This method must be called with ParallelGCImpl.mutex locked + */ + void push(Pointer ptr) { assert top < SIZE; buffer[top++] = ptr; - ParallelGCImpl.mutex.unlock(); } - HeapChunk.Header pop() { + Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index b2c92634b1ed..d5c98447af63 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -7,6 +7,7 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; +import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; @@ -78,7 +79,7 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { allocChunkTL.set(chunk); } - public void push(HeapChunk.Header ptr) { + public void push(Pointer ptr) { buffer.push(ptr); } @@ -170,10 +171,15 @@ private void waitForIdleImpl() { private void drainBuffer() { debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); do { - HeapChunk.Header chunk; - while ((chunk = buffer.pop()).notEqual(WordFactory.nullPointer())) { - debugLog().string("WW drain chunk=").zhex(chunk).newline(); - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) chunk, getVisitor()); + Pointer ptr; + while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { + debugLog().string("WW drain chunk=").zhex(ptr).newline(); + if (ptr.and(0x01).notEqual(0)) { + // unaligned chunk + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); + } else { + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + } } AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); debugLog().string("WW drain tlab=").zhex(tlab).newline(); From ec5dd99ad7793c042879f96bce1b8804e55fcae1 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 19 Sep 2022 13:27:06 +0300 Subject: [PATCH 60/99] HyperAlloc works but load is badly unbalanced --- .../com/oracle/svm/core/genscavenge/Space.java | 2 +- .../core/genscavenge/parallel/ChunkBuffer.java | 3 +-- .../genscavenge/parallel/ParallelGCImpl.java | 16 ++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0f265f52c7f5..651da2242633 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -511,7 +511,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01)); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 69e6779d514e..c37fa9042790 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,12 +1,11 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; /** - * Synchronized buffer where chunks to be scanned are stored. + * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { private static final int SIZE = 10 * 1024; ///handle overflow diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d5c98447af63..2431bbb5bff6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -170,11 +170,11 @@ private void waitForIdleImpl() { private void drainBuffer() { debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); - do { + while (true) { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(0x01).notEqual(0)) { + if (ptr.and(0x01).notEqual(0)) {///magic // unaligned chunk UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); } else { @@ -183,13 +183,17 @@ private void drainBuffer() { } AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); debugLog().string("WW drain tlab=").zhex(tlab).newline(); - if (tlab.notEqual(WordFactory.nullPointer())) { + if (tlab.equal(WordFactory.nullPointer())) { + break; + } else { scannedChunkTL.set(tlab); AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); + if (allocChunkTL.get().equal(tlab)) { + // this tlab is now black, retire it + allocChunkTL.set(WordFactory.nullPointer()); + } } - } while (buffer.size() > 0 || allocChunkTL.get().notEqual(scannedChunkTL.get())); - - allocChunkTL.set(WordFactory.nullPointer()); + } scannedChunkTL.set(WordFactory.nullPointer()); } From d59cb3646c73f81846c05dc90c7f8c2a6a676fd6 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 19 Sep 2022 18:02:50 +0300 Subject: [PATCH 61/99] Report stats for relevant threads only --- .../com/oracle/svm/core/genscavenge/parallel/Stats.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java index a08cbd0c1ae7..6a8a1dcd9bf5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java @@ -71,9 +71,10 @@ public void print(Log log) { totalStats.reset(); log.string("PGC stats:").newline(); for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - StatsImpl stats = statsTL.get(vmThread); - if (stats != null) { - Thread jt = PlatformThreads.fromVMThread(vmThread); + Thread jt = PlatformThreads.fromVMThread(vmThread); + String jtName = jt.getName(); + if (jt == Thread.currentThread() || jtName != null && jtName.startsWith("ParallelGCWorker-")) { + StatsImpl stats = statsTL.get(vmThread); log.string(" ").string(jt.getName()).string(":").newline(); totalStats.objectVisitTime += stats.objectVisitTime; totalStats.allocTime += stats.allocTime; From c4c4b34768959490d94096fd10746f6087aed807 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 13:17:50 +0300 Subject: [PATCH 62/99] Fixed SerialGC compilation --- .../src/com/oracle/svm/core/genscavenge/Space.java | 13 +++++++------ .../core/genscavenge/parallel/ParallelGCImpl.java | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 651da2242633..88f7800d4fa0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -190,7 +190,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer freeMemory(UnsignedWord objectSize) { - assert ParallelGCImpl.isInParallelPhase(); + assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); @@ -378,7 +378,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -392,6 +392,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); + assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -481,12 +482,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); ParallelGCImpl.mutex.unlock(); } @@ -505,12 +506,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 2431bbb5bff6..a19fa1dfd294 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -60,7 +60,7 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; + return singleton().inParallelPhase; } public static void waitForIdle() { From c061049b0825d7974e94c622de5d14a5d01ccc8f Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 16:18:06 +0300 Subject: [PATCH 63/99] Cleanup --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 3 --- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 10 +++++----- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b66b85a79047..95b27b51c1e4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -370,9 +370,6 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); - boolean compressed = ReferenceAccess.singleton().haveCompressedReferences();///rm - boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); - verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 88f7800d4fa0..266f5e6b447c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -512,7 +512,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index a19fa1dfd294..029079c72798 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -28,6 +28,7 @@ public class ParallelGCImpl extends ParallelGC { + public static final int UNALIGNED_BIT = 0x01; private static final int MAX_WORKERS_COUNT = 31; /** @@ -88,7 +89,7 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker + workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); for (int i = 1; i < workerCount; i++) { // We reuse the gc thread for worker #0 @@ -174,9 +175,8 @@ private void drainBuffer() { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(0x01).notEqual(0)) {///magic - // unaligned chunk - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); } @@ -209,7 +209,7 @@ private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } - static Log debugLog() {///rm + static Log debugLog() { return Log.noopLog(); } } From 1dbd6c104ef8b103985620f2cb9abcfd5e647eb3 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 17:14:03 +0300 Subject: [PATCH 64/99] Determine buffer size at runtime --- .../genscavenge/parallel/ChunkBuffer.java | 30 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 16 ++++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index c37fa9042790..72480544e776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,6 +1,6 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.log.Log; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -8,27 +8,30 @@ * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { - private static final int SIZE = 10 * 1024; ///handle overflow - private static final Pointer[] buffer = new Pointer[SIZE]; - private int top; + private Pointer buffer; + private int size, top, step; - private Log debugLog() { - return ParallelGCImpl.debugLog(); + ChunkBuffer(int maxChunks, int refSize) { + this.step = refSize; + this.size = maxChunks * refSize; + this.buffer = UnmanagedMemory.malloc(this.size); } /** * This method must be called with ParallelGCImpl.mutex locked */ void push(Pointer ptr) { - assert top < SIZE; - buffer[top++] = ptr; + assert top < size; + buffer.writeWord(top, ptr); + top += step; } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - return buffer[--top]; + top -= step; + return buffer.readWord(top); } else { return WordFactory.nullPointer(); } @@ -36,13 +39,4 @@ Pointer pop() { ParallelGCImpl.mutex.unlock(); } } - - int size() { - ParallelGCImpl.mutex.lock(); - try { - return top; - } finally { - ParallelGCImpl.mutex.unlock(); - } - } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 029079c72798..844f9945328c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,7 +6,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; -import com.oracle.svm.core.genscavenge.HeapChunk; +import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; @@ -21,7 +21,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; @@ -39,8 +38,6 @@ public class ParallelGCImpl extends ParallelGC { private int allWorkersBusy; private final AtomicInteger busyWorkers = new AtomicInteger(0); - private static final ChunkBuffer buffer = new ChunkBuffer(); - /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -53,6 +50,7 @@ public class ParallelGCImpl extends ParallelGC { private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); + private ChunkBuffer buffer; private volatile boolean inParallelPhase; @Fold @@ -89,10 +87,17 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); + // Allocate buffer large enough to store maximum possible number of heap chunks + long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); + long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); + long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); + long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; + buffer = new ChunkBuffer((int) maxChunks, ConfigurationValues.getObjectLayout().getReferenceSize()); + workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); + // We reuse the gc thread for worker #0 for (int i = 1; i < workerCount; i++) { - // We reuse the gc thread for worker #0 startWorkerThread(i); } } @@ -170,7 +175,6 @@ private void waitForIdleImpl() { } private void drainBuffer() { - debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); while (true) { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { From ca5edacade095b3e5df87ac5e9154b61ece62893 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 17:19:46 +0300 Subject: [PATCH 65/99] Removed Stats --- .../genscavenge/parallel/ParallelGCImpl.java | 5 - .../svm/core/genscavenge/parallel/Stats.java | 205 ------------------ .../parallel/ThreadLocalBuffer.java | 61 ------ 3 files changed, 271 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 844f9945328c..bcd7c615cafc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -121,7 +121,6 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); - getStats().install(); while (true) { try { do { @@ -205,10 +204,6 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static Stats getStats() { - return Stats.stats(); - } - private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java deleted file mode 100644 index 6a8a1dcd9bf5..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.PlatformThreads; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalObject; -import org.graalvm.nativeimage.IsolateThread; - -public abstract class Stats { - private static final Stats realStats = new RealStats(); - private static final Stats noopStats = new NoopStats(); - - static Stats stats() { - return realStats; - } - - static Stats noopStats() { - return noopStats; - } - - public void reset() {} - public void print(Log log) {} - void install() {} - public void noteAllocTime(long t) {} - public void noteForwardInstallTime(long t) {} - public void noteObjectVisitTime(long t) {} - public void noteQueueCopyTime(long t) {} - public void noteQueueScanTime(long t) {} - public void noteLostObject() {} - void notePut(int putIndex, int getIndex, int capacity) {} - void noteGet() {} -} - -class NoopStats extends Stats { -} - -class RealStats extends Stats { - private static final FastThreadLocalObject statsTL = - FastThreadLocalFactory.createObject(StatsImpl.class, "ParGC.Stats"); - - private final StatsImpl defaultStats = new StatsImpl(); - private final StatsImpl totalStats = new StatsImpl(); - - private StatsImpl impl() { - StatsImpl stats = statsTL.get(); - if (stats == null) { - stats = defaultStats; - statsTL.set(stats); - } - return stats; - } - - @Override - void install() { - statsTL.set(new StatsImpl()); - } - - @Override - public void reset() { - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - StatsImpl stats = statsTL.get(vmThread); - if (stats != null) { - stats.reset(); - } - } - } - - @Override - public void print(Log log) { - totalStats.reset(); - log.string("PGC stats:").newline(); - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - Thread jt = PlatformThreads.fromVMThread(vmThread); - String jtName = jt.getName(); - if (jt == Thread.currentThread() || jtName != null && jtName.startsWith("ParallelGCWorker-")) { - StatsImpl stats = statsTL.get(vmThread); - log.string(" ").string(jt.getName()).string(":").newline(); - totalStats.objectVisitTime += stats.objectVisitTime; - totalStats.allocTime += stats.allocTime; - totalStats.forwardInstallTime += stats.forwardInstallTime; - totalStats.queueScanTime += stats.queueScanTime; - totalStats.queueCopyTime += stats.queueCopyTime; - totalStats.lostObjectCount += stats.lostObjectCount; - totalStats.putCount += stats.putCount; - totalStats.getCount += stats.getCount; - totalStats.maxQueueSize = Math.max(totalStats.maxQueueSize, stats.maxQueueSize); - stats.print(log); - } - } - log.string(" total:").newline(); - totalStats.print(log); - } - - @Override - public void noteAllocTime(long t) { - impl().noteAllocTime(t); - } - - @Override - public void noteForwardInstallTime(long t) { - impl().noteForwardInstallTime(t); - } - - @Override - public void noteObjectVisitTime(long t) { - impl().noteObjectVisitTime(t); - } - - @Override - public void noteQueueCopyTime(long t) { - impl().noteQueueCopyTime(t); - } - - @Override - public void noteQueueScanTime(long t) { - impl().noteQueueScanTime(t); - } - - @Override - public void noteLostObject() { - impl().noteLostObject(); - } - - @Override - void notePut(int putIndex, int getIndex, int capacity) { - impl().notePut(putIndex, getIndex, capacity); - } - - @Override - public void noteGet() { - impl().noteGet(); - } -} - -class StatsImpl extends Stats { - int allocTime, forwardInstallTime, objectVisitTime, queueCopyTime, queueScanTime; - int putCount, getCount, lostObjectCount, maxQueueSize; - - @Override - public void reset() { - allocTime = forwardInstallTime = objectVisitTime = queueCopyTime = queueScanTime = 0; - putCount = getCount = lostObjectCount = maxQueueSize = 0; - } - - @Override - public void print(Log log) { - log.string(" visit ").unsigned(objectVisitTime) - .string(" alloc ").unsigned(allocTime) - .string(" fwptr ").unsigned(forwardInstallTime) - .string(" qscan ").unsigned(queueScanTime) - .string(" qcopy ").unsigned(queueCopyTime) - .string(" lost ").unsigned(lostObjectCount).newline(); - log.string(" put ").unsigned(putCount) - .string(" get ").unsigned(getCount) - .string(" max ").unsigned(maxQueueSize).newline(); - } - - @Override - public void noteAllocTime(long t) { - allocTime += t; - } - - @Override - public void noteForwardInstallTime(long t) { - forwardInstallTime += t; - } - - @Override - public void noteObjectVisitTime(long t) { - objectVisitTime += t; - } - - @Override - public void noteQueueCopyTime(long t) { - queueCopyTime += t; - } - - @Override - public void noteQueueScanTime(long t) { - queueScanTime += t; - } - - @Override - public void noteLostObject() { - lostObjectCount++; - } - - @Override - void notePut(int putIndex, int getIndex, int capacity) { - int size = putIndex - getIndex; - if (size < 0) { - size += capacity; - } - if (size > maxQueueSize) { - maxQueueSize = size; - } - putCount++; - } - - @Override - void noteGet() { - getCount++; - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java deleted file mode 100644 index 1dc025567def..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.heap.OutOfMemoryUtil; -import com.oracle.svm.core.log.Log; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -/** - * Thread local (non MT safe) buffer where pointers to grey objects are stored. - */ -public class ThreadLocalBuffer { - private static final int INITIAL_SIZE = 128 * 1024 * 8; // 128k words - private static final OutOfMemoryError OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate a ThreadLocalBuffer"); - - private Pointer buffer = WordFactory.nullPointer(); - private UnsignedWord size = WordFactory.unsigned(INITIAL_SIZE); - private UnsignedWord top = WordFactory.zero(); - - private Log log() { - return Log.log(); - } - - void runtimeInit() { - buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); - ensureBuffer(); - } - - private void ensureCapacity() { - if (top.aboveOrEqual(size)) { - size = size.multiply(2); - buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, size); - ensureBuffer(); - log().string("TLB grow to ").unsigned(size).newline(); - } - } - - private void ensureBuffer() { - if (buffer.isNull()) { - throw OutOfMemoryUtil.reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); - } - } - - void push(Pointer ptr) { - ensureCapacity(); - buffer.writeWord(top, ptr); - top = top.add(8); - } - - Object pop() { - if (top.aboveThan(0)) { - top = top.subtract(8); - Pointer ptr = buffer.readWord(top); - return ptr.toObjectNonNull(); - } else { - return null; - } - } -} \ No newline at end of file From 757075ffbc4c59961c853d156f61cc9cba8c441c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 4 Oct 2022 19:07:14 +0300 Subject: [PATCH 66/99] Cleanup --- .../oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 870494eb4f1a..88adc11c4a0b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,14 +24,11 @@ */ package com.oracle.svm.core.genscavenge; -import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; From 01627fce89cedc3c6dc43dba303cc94ca6c427d1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 2 Nov 2022 10:47:20 +0600 Subject: [PATCH 67/99] Fixed crash due to unscanned chunks HyperAlloc works for hours with heap verification --- .../src/com/oracle/svm/core/genscavenge/Space.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 266f5e6b447c..ffe6a8e7fe85 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -487,9 +487,11 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - ParallelGCImpl.mutex.unlock(); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } if (this.isOldSpace()) { @@ -511,9 +513,11 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - ParallelGCImpl.mutex.unlock(); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } if (this.isOldSpace()) { From 80df70ec10a8e89952e1808818322e58dbad6274 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 3 Nov 2022 14:37:39 +0600 Subject: [PATCH 68/99] Added copyright notices --- .../genscavenge/parallel/ChunkBuffer.java | 26 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 26 +++++++++++++++++++ .../parallel/ParallelGCOptions.java | 26 +++++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 26 +++++++++++++++++++ 4 files changed, 104 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 72480544e776..205bed8a1d3a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import org.graalvm.nativeimage.UnmanagedMemory; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index bcd7c615cafc..15b1d660f15d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 208f06fc633e..05002102799f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.option.RuntimeOptionKey; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 3bd3cf42b34a..aa1af21c497a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateOptions; From 2a540d303e5c99b3ce958f993c72f08407144b80 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 10 Nov 2022 15:39:20 +0300 Subject: [PATCH 69/99] Fixed merge errors --- .../core/genscavenge/parallel/ParallelGCImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 15b1d660f15d..e6e4f86fbd91 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -27,8 +27,9 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; -import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -45,7 +46,8 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -239,8 +241,10 @@ static Log debugLog() { } } -@AutomaticFeature -class ParallelGCFeature implements Feature { +@Platforms(Platform.HOSTED_ONLY.class) +@AutomaticallyRegisteredFeature() +@SuppressWarnings("unused") +class ParallelGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { From f228f74322a147d35d8df1fc3c38d42bbe31311b Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 10 Nov 2022 16:31:41 +0300 Subject: [PATCH 70/99] Fixed Substrate options to work with ParallelGC --- .../core/genscavenge/JfrGCEventSupport.java | 2 +- .../SerialAndEpsilonGCOptions.java | 22 +++++++++---------- .../genscavenge/graal/BarrierSnippets.java | 2 +- .../graal/GenScavengeGCFeature.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 5 +++++ .../heap/Target_java_lang_ref_Reference.java | 6 ++--- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 5c813ee3c3fc..5270d0282244 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -141,7 +141,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue(); + return SubstrateOptions.useSerialOrParallelGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index e3179517289d..bc4cfb993692 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -40,13 +40,13 @@ /** Common options that can be specified for both the serial and the epsilon GC. */ public final class SerialAndEpsilonGCOptions { @Option(help = "The maximum heap size as percent of physical memory. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "The size of an aligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly) { + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -59,28 +59,28 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * fit in an aligned chunk. */ @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. 0 implies (AlignedHeapChunkSize / 8). Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Fill unused memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Before use, fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "After use, Fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::graalCeGCOnly); private SerialAndEpsilonGCOptions() { } - public static void serialOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue()) { + private static void graalCeGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useGraalCeGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 247da5e8440e..69adc25a7743 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -204,7 +204,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useSerialOrParallelGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index e311f8b0ccdf..f44e3ec53a28 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -155,7 +155,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.UseSerialGC.getValue()) { + if (SubstrateOptions.useSerialOrParallelGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 3fd681029fd8..91bebd8810c5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -338,6 +338,11 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; + @Fold + public static boolean useSerialOrParallelGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue(); + } + @Fold public static boolean useGraalCeGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index de86cf928f8c..3c2cf2e3ca6a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -95,7 +95,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotSerialNotEpsilonGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotGraalCeGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -229,9 +229,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotSerialNotEpsilonGC implements BooleanSupplier { +class NotGraalCeGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue(); + return !SubstrateOptions.useGraalCeGC(); } } From 3cbe52e21055ca83835d9f9c58fcc31fe21d2732 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 Nov 2022 16:00:42 +0300 Subject: [PATCH 71/99] Post-review renaming and cleanup --- .../core/genscavenge/AlignedHeapChunk.java | 4 +-- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../core/genscavenge/PinnedObjectImpl.java | 2 +- .../SerialAndEpsilonGCOptions.java | 22 ++++++------- .../oracle/svm/core/genscavenge/Space.java | 31 +++++++++---------- .../core/genscavenge/UnalignedHeapChunk.java | 2 +- ...GC.java => UseMarkAndCopyOrEpsilonGC.java} | 4 +-- .../graal/GenScavengeGCFeature.java | 3 +- .../genscavenge/parallel/ChunkBuffer.java | 19 ++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 22 ++++++------- .../parallel/ParallelGCOptions.java | 6 ++-- .../com/oracle/svm/core/SubstrateOptions.java | 19 +++++++++--- .../com/oracle/svm/core/heap/ParallelGC.java | 6 ++-- .../heap/Target_java_lang_ref_Reference.java | 6 ++-- .../oracle/svm/core/hub/LayoutEncoding.java | 1 + 16 files changed, 82 insertions(+), 70 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{UseGraalCeGC.java => UseMarkAndCopyOrEpsilonGC.java} (92%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 0150e3c5add0..ba0180117ac8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -113,7 +113,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { } /** Retract the latest allocation. Used by parallel collector. */ - static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { + static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); assert newTop.aboveThan(HeapChunk.asPointer(that)); HeapChunk.setTopPointer(that, newTop); @@ -160,7 +160,7 @@ static MemoryWalker.HeapChunkAccess getMemoryWal } /** Methods for a {@link MemoryWalker} to access an aligned heap chunk. */ - @AutomaticallyRegisteredImageSingleton(onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index d7fee4f8a074..dbc9ea720977 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -74,7 +74,6 @@ import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; @@ -1043,7 +1042,7 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.waitForIdle(); } else if (isIncremental) { scanGreyObjectsLoop(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index fd7c91f59c27..29617435e89f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -878,7 +878,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseGraalCeGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index ab349ead749b..a5686c28b8f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -42,7 +42,7 @@ /** Support for pinning objects to a memory address with {@link PinnedObject}. */ final class PinnedObjectImpl implements PinnedObject { - @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) static class PinnedObjectSupportImpl implements PinnedObjectSupport { @Override public PinnedObject create(Object object) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index bc4cfb993692..e6b76ab52316 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -40,13 +40,13 @@ /** Common options that can be specified for both the serial and the epsilon GC. */ public final class SerialAndEpsilonGCOptions { @Option(help = "The maximum heap size as percent of physical memory. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "The size of an aligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly) { + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -59,28 +59,28 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * fit in an aligned chunk. */ @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. 0 implies (AlignedHeapChunkSize / 8). Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Fill unused memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Before use, fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "After use, Fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); private SerialAndEpsilonGCOptions() { } - private static void graalCeGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.useGraalCeGC()) { + private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.UseMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ce980f706f76..bdc75c7fef3f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -48,7 +48,6 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; /** * A Space is a collection of HeapChunks. @@ -170,7 +169,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk; - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { oldChunk = ParallelGCImpl.getThreadLocalChunk(); } else { oldChunk = getLastAlignedHeapChunk(); @@ -189,22 +188,22 @@ private Pointer allocateMemory(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - private Pointer freeMemory(UnsignedWord objectSize) { - assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + private Pointer retractAllocation(UnsignedWord objectSize) { + assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); - return AlignedHeapChunk.freeMemory(oldChunk, objectSize); + return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.mutex.lock(); if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.mutex.unlock(); ParallelGCImpl.setThreadLocalChunk(newChunk); } @@ -231,7 +230,7 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); } appendAlignedHeapChunkUninterruptibly(aChunk); @@ -284,7 +283,7 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); } appendUnalignedHeapChunkUninterruptibly(uChunk); @@ -378,7 +377,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -392,7 +391,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); - assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -438,7 +437,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { return copy; } else { // Retract speculatively allocated memory - freeMemory(size); + retractAllocation(size); return forward; } } @@ -482,12 +481,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported()) { + if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); @@ -508,12 +507,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported()) { + if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 16bb628bc1b8..6f5f1956cf1e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -162,7 +162,7 @@ public static MemoryWalker.HeapChunkAccess g return ImageSingletons.lookup(UnalignedHeapChunk.MemoryWalkerAccessImpl.class); } - @AutomaticallyRegisteredImageSingleton(onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java similarity index 92% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java index 8da5cca783c0..9e824c594140 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java @@ -32,9 +32,9 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -public class UseGraalCeGC implements BooleanSupplier { +public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.useGraalCeGC(); + return SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index f44e3ec53a28..e9efa8939ccf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; +import com.oracle.svm.core.genscavenge.UseMarkAndCopyOrEpsilonGC; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -69,7 +70,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return new com.oracle.svm.core.genscavenge.UseGraalCeGC().getAsBoolean(); + return new UseMarkAndCopyOrEpsilonGC().getAsBoolean(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 205bed8a1d3a..b1bb86500d86 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,8 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.config.ConfigurationValues; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -35,28 +37,25 @@ */ public class ChunkBuffer { private Pointer buffer; - private int size, top, step; + private int size, top; - ChunkBuffer(int maxChunks, int refSize) { - this.step = refSize; - this.size = maxChunks * refSize; + ChunkBuffer(int maxChunks) { + this.size = maxChunks * ConfigurationValues.getTarget().wordSize; this.buffer = UnmanagedMemory.malloc(this.size); } - /** - * This method must be called with ParallelGCImpl.mutex locked - */ void push(Pointer ptr) { + ParallelGCImpl.mutex.assertIsOwner("Must hold ParallelGCImpl.mutex"); assert top < size; buffer.writeWord(top, ptr); - top += step; + top += ConfigurationValues.getTarget().wordSize; } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - top -= step; + top -= ConfigurationValues.getTarget().wordSize; return buffer.readWord(top); } else { return WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e6e4f86fbd91..bd2464d09c2a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -120,7 +120,7 @@ public void startWorkerThreadsImpl() { long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; - buffer = new ChunkBuffer((int) maxChunks, ConfigurationValues.getObjectLayout().getReferenceSize()); + buffer = new ChunkBuffer((int) maxChunks); workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); @@ -131,7 +131,7 @@ public void startWorkerThreadsImpl() { } private int getWorkerCount() { - int setting = ParallelGCOptions.ParallelGCWorkers.getValue(); + int setting = ParallelGCOptions.ParallelGCThreads.getValue(); int workers = setting > 0 ? setting : getDefaultWorkerCount(); workers = Math.min(workers, MAX_WORKERS_COUNT); verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); @@ -212,15 +212,15 @@ private void drainBuffer() { AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); } } - AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); - debugLog().string("WW drain tlab=").zhex(tlab).newline(); - if (tlab.equal(WordFactory.nullPointer())) { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + debugLog().string("WW drain allocChunk=").zhex(allocChunk).newline(); + if (allocChunk.equal(WordFactory.nullPointer())) { break; } else { - scannedChunkTL.set(tlab); - AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); - if (allocChunkTL.get().equal(tlab)) { - // this tlab is now black, retire it + scannedChunkTL.set(allocChunk); + AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + if (allocChunkTL.get().equal(allocChunk)) { + // this allocation chunk is now black, retire it allocChunkTL.set(WordFactory.nullPointer()); } } @@ -248,7 +248,7 @@ class ParallelGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return ParallelGC.isSupported(); + return ParallelGC.isEnabled(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 05002102799f..6fe4b04d6ae9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,8 +30,10 @@ import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionType; +import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.Immutable; + public class ParallelGCOptions { @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) - public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0); + public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 91bebd8810c5..556b2fe13a46 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -37,8 +37,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Set; import java.util.function.Predicate; +import com.oracle.svm.core.util.InterruptImageBuilding; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -326,25 +328,34 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// @Option(help = "Use a parallel GC")// - public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false) { + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::validateParallelGC) { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { SubstrateOptions.UseSerialGC.update(values, false); SubstrateOptions.UseEpsilonGC.update(values, false); - ConcealedOptions.UseRememberedSet.update(values, false); - SubstrateOptions.MultiThreaded.update(values, true); } } }; + private static void validateParallelGC(OptionKey unused) { + validateParallelGC(SubstrateOptions.MultiThreaded, true); + validateParallelGC(ConcealedOptions.UseRememberedSet, false); + } + + private static void validateParallelGC(HostedOptionKey optionKey, boolean expected) { + if (optionKey.getValue() != expected) { + throw UserError.abort("ParallelGC requires the option %s to be %s", optionKey, expected); + } + } + @Fold public static boolean useSerialOrParallelGC() { return UseSerialGC.getValue() || UseParallelGC.getValue(); } @Fold - public static boolean useGraalCeGC() { + public static boolean UseMarkAndCopyOrEpsilonGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index aa1af21c497a..95d8de510c9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,12 +33,12 @@ public abstract class ParallelGC { @Fold - public static boolean isSupported() { + public static boolean isEnabled() { return SubstrateOptions.UseParallelGC.getValue(); } public static void startWorkerThreads() { - if (isSupported()) { + if (isEnabled()) { ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index 3c2cf2e3ca6a..2a3fdd519d37 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -95,7 +95,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotGraalCeGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotMarkAndCopyOrEpsilonGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -229,9 +229,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotGraalCeGC implements BooleanSupplier { +class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.useGraalCeGC(); + return !SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index a60463fdcdbb..b290969c630a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -274,6 +274,7 @@ public static UnsignedWord getSizeFromObjectInline(Object obj) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { + assert encoding == KnownIntrinsics.readHub(obj).getLayoutEncoding(); if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From c8ae8a3b317da18b6977f31726010bf0cc4912ba Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 22 Nov 2022 15:39:42 +0300 Subject: [PATCH 72/99] Review: forwarding header installation was not atomic enough --- .../core/genscavenge/ObjectHeaderImpl.java | 13 +++---------- .../oracle/svm/core/genscavenge/Space.java | 19 +++++++++---------- .../genscavenge/parallel/ParallelGCImpl.java | 3 --- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index be3dc1f0e23c..4146e121a578 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -358,17 +358,10 @@ static Object installForwardingPointerParallel(Object original, UnsignedWord ori // try installing the new header Pointer originalPtr = Word.objectToUntrackedPointer(original); - long witness; - if (getReferenceSize() == Integer.BYTES) { - witness = originalPtr.compareAndSwapInt(getHubOffset(), - (int) originalHeader.rawValue(), (int) forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); - } else { - witness = originalPtr.compareAndSwapLong(getHubOffset(), - originalHeader.rawValue(), forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); - } + UnsignedWord witness = originalPtr.compareAndSwapWord(getHubOffset(), originalHeader, forwardHeader, LocationIdentity.ANY_LOCATION); assert isPointerToForwardedObject(originalPtr); - if (witness != originalHeader.rawValue()) { - return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); + if (witness.notEqual(originalHeader)) { + return getForwardedObject(originalPtr, witness); } else if (hasShift) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index bdc75c7fef3f..b5d09eaabe72 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,6 +28,7 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.config.ConfigurationValues; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -395,7 +396,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); - UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + int hubOffset = ObjectHeaderImpl.getHubOffset(); + UnsignedWord originalHeader = originalMemory.readWord(hubOffset); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } @@ -414,19 +416,16 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } // Install forwarding pointer into the original header - assert ObjectHeaderImpl.getHubOffset() == 0; Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); - if (forward == copy) { // We have won the race, now we must copy the object bits. First install the original header - int headerSize = ObjectHeaderImpl.getReferenceSize(); - if (headerSize == Integer.BYTES) { - copyMemory.writeInt(0, (int) originalHeader.rawValue()); - } else { - copyMemory.writeWord(0, originalHeader); - } + copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object - UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); + if (hubOffset > 0) { + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); + } + int offset = hubOffset + ConfigurationValues.getTarget().wordSize; + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index bd2464d09c2a..168725c472e4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -112,9 +112,6 @@ public void push(Pointer ptr) { @Override public void startWorkerThreadsImpl() { - int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); - VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - // Allocate buffer large enough to store maximum possible number of heap chunks long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); From 1a22815ba06b0cc42d88993a1a54e2a9798d71de Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 23 Nov 2022 15:17:00 +0300 Subject: [PATCH 73/99] Worker thread shutdown --- .../oracle/svm/core/genscavenge/HeapImpl.java | 5 +++++ .../genscavenge/parallel/ParallelGCImpl.java | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 29617435e89f..a176b61d6442 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.List; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.SuppressFBWarnings; @@ -200,6 +202,9 @@ boolean walkMemory(MemoryWalker.Visitor visitor) { @Override @Uninterruptible(reason = "Tear-down in progress.") public boolean tearDown() { + if (ParallelGC.isEnabled()) { + ParallelGCImpl.singleton().tearDown(); + } youngGeneration.tearDown(); oldGeneration.tearDown(); getChunkProvider().tearDown(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 168725c472e4..1bb512dbd632 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -27,7 +27,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; -import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; @@ -40,6 +40,7 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; @@ -51,7 +52,9 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -64,6 +67,7 @@ public class ParallelGCImpl extends ParallelGC { */ private int workerCount = 1; private int allWorkersBusy; + private List workers; private final AtomicInteger busyWorkers = new AtomicInteger(0); /** @@ -120,11 +124,9 @@ public void startWorkerThreadsImpl() { buffer = new ChunkBuffer((int) maxChunks); workerCount = getWorkerCount(); - allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); - // We reuse the gc thread for worker #0 - for (int i = 1; i < workerCount; i++) { - startWorkerThread(i); - } + // We reuse the gc thread as the last worker + workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); + allWorkersBusy = ~(-1 << (workerCount - 1)); } private int getWorkerCount() { @@ -141,7 +143,7 @@ private int getDefaultWorkerCount() { return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } - private void startWorkerThread(int n) { + private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); @@ -177,6 +179,12 @@ private void startWorkerThread(int n) { t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); + return t; + } + + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + public void tearDown() { + workers.forEach(PlatformThreads::exit); } private void waitForIdleImpl() { From 00a8f37e34749c28eca73ab7f228d2c8b17a93f1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 23 Nov 2022 15:56:20 +0300 Subject: [PATCH 74/99] Made `ChunkBuffer` grow as needed and free memory in the end --- .../genscavenge/parallel/ChunkBuffer.java | 52 ++++++++++++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 9 +--- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index b1bb86500d86..60a1614ded48 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -26,9 +26,11 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.os.CommittedMemoryProvider; import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -36,26 +38,38 @@ * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { + private static final int INITIAL_SIZE = 1024 * wordSize(); + private Pointer buffer; private int size, top; - ChunkBuffer(int maxChunks) { - this.size = maxChunks * ConfigurationValues.getTarget().wordSize; - this.buffer = UnmanagedMemory.malloc(this.size); + @Fold + static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + + ChunkBuffer() { + this.size = INITIAL_SIZE; + this.buffer = malloc(this.size); } void push(Pointer ptr) { - ParallelGCImpl.mutex.assertIsOwner("Must hold ParallelGCImpl.mutex"); - assert top < size; + assert !ParallelGCImpl.isInParallelPhase() || ParallelGCImpl.mutex.isOwner(); + if (top >= size) { + int oldSize = size; + size *= 2; + assert top < size; + buffer = realloc(buffer, oldSize, size); + } buffer.writeWord(top, ptr); - top += ConfigurationValues.getTarget().wordSize; + top += wordSize(); } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - top -= ConfigurationValues.getTarget().wordSize; + top -= wordSize(); return buffer.readWord(top); } else { return WordFactory.nullPointer(); @@ -64,4 +78,26 @@ Pointer pop() { ParallelGCImpl.mutex.unlock(); } } + + void release() { + free(buffer, size); + } + + @AlwaysInline("GC performance") + private Pointer malloc(int size) { + return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(size)); + } + + @AlwaysInline("GC performance") + private Pointer realloc(Pointer orig, int origSize, int newSize) { + Pointer ptr = malloc(newSize); + UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); + free(orig, origSize); + return ptr; + } + + @AlwaysInline("GC performance") + private void free(Pointer ptr, int size) { + CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(size)); + } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1bb512dbd632..0349b4293131 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -116,13 +116,7 @@ public void push(Pointer ptr) { @Override public void startWorkerThreadsImpl() { - // Allocate buffer large enough to store maximum possible number of heap chunks - long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); - long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); - long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); - long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; - buffer = new ChunkBuffer((int) maxChunks); - + buffer = new ChunkBuffer(); workerCount = getWorkerCount(); // We reuse the gc thread as the last worker workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); @@ -184,6 +178,7 @@ private Thread startWorkerThread(int n) { @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { + buffer.release(); workers.forEach(PlatformThreads::exit); } From 2af8a3064bc4c0322aa831efcfb5e73edb3b63a2 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 25 Nov 2022 12:56:22 +0300 Subject: [PATCH 75/99] More robust worker thread routine --- .../genscavenge/parallel/ParallelGCImpl.java | 114 ++++++++---------- 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0349b4293131..dd1668e2353e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; @@ -59,16 +60,9 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private static final int MAX_WORKERS_COUNT = 31; - /** - * Number of parallel GC workers. Default is 1 so that if GC happens before worker threads are started, - * it occurs on a single thread just like serial GC. - */ - private int workerCount = 1; - private int allWorkersBusy; private List workers; - private final AtomicInteger busyWorkers = new AtomicInteger(0); + private final AtomicInteger busyWorkers = new AtomicInteger(); /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -112,21 +106,22 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { public void push(Pointer ptr) { buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); + } } @Override public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); - workerCount = getWorkerCount(); - // We reuse the gc thread as the last worker + int workerCount = getWorkerCount(); + busyWorkers.set(workerCount); workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); - allWorkersBusy = ~(-1 << (workerCount - 1)); } private int getWorkerCount() { int setting = ParallelGCOptions.ParallelGCThreads.getValue(); int workers = setting > 0 ? setting : getDefaultWorkerCount(); - workers = Math.min(workers, MAX_WORKERS_COUNT); verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); return workers; } @@ -144,27 +139,24 @@ private Thread startWorkerThread(int n) { while (true) { try { - do { - debugLog().string("WW block ").unsigned(n).newline(); + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { + if (busyWorkers.decrementAndGet() == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); mutex.lock(); parPhase.block(); mutex.unlock(); - } while ((busyWorkers.get() & (1 << n)) == 0); + busyWorkers.incrementAndGet(); + debugLog().string("WW run ").unsigned(n).newline(); + } - debugLog().string("WW run ").unsigned(n).newline(); - drainBuffer(); - int witness = busyWorkers.get(); - int expected; do { - expected = witness; - witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); - } while (witness != expected); - if (busyWorkers.get() == 0) { - mutex.lock(); - seqPhase.signal(); - mutex.unlock(); - } - debugLog().string("WW idle ").unsigned(n).newline(); + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); } catch (Throwable ex) { VMError.shouldNotReachHere(ex); } @@ -176,53 +168,43 @@ private Thread startWorkerThread(int n) { return t; } - @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) - public void tearDown() { - buffer.release(); - workers.forEach(PlatformThreads::exit); - } - private void waitForIdleImpl() { - inParallelPhase = true; + assert allocChunkTL.get().isNonNull(); + push(HeapChunk.asPointer(allocChunkTL.get())); + allocChunkTL.set(WordFactory.nullPointer()); debugLog().string("PP start workers\n"); - busyWorkers.set(allWorkersBusy); + inParallelPhase = true; parPhase.broadcast(); // let worker threads run - drainBuffer(); - mutex.lock(); - while (busyWorkers.get() != 0) { - debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + while (inParallelPhase) { + debugLog().string("PP wait\n"); seqPhase.block(); // wait for them to become idle } mutex.unlock(); - - inParallelPhase = false; } - private void drainBuffer() { - while (true) { - Pointer ptr; - while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { - debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(UNALIGNED_BIT).notEqual(0)) { - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); - } else { - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); - } - } - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - debugLog().string("WW drain allocChunk=").zhex(allocChunk).newline(); - if (allocChunk.equal(WordFactory.nullPointer())) { - break; + private void scanChunk(Pointer ptr) { + if (ptr.isNonNull()) { + debugLog().string("WW scan chunk=").zhex(ptr).newline(); + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { - scannedChunkTL.set(allocChunk); - AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); - if (allocChunkTL.get().equal(allocChunk)) { - // this allocation chunk is now black, retire it - allocChunkTL.set(WordFactory.nullPointer()); - } + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + } + } + } + + private void scanAllocChunk() { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + if (allocChunk.isNonNull()) { + debugLog().string("WW scan alloc=").zhex(allocChunk).newline(); + scannedChunkTL.set(allocChunk); + AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + if (allocChunkTL.get().equal(allocChunk)) { + // this allocation chunk is now black, retire it + allocChunkTL.set(WordFactory.nullPointer()); } } scannedChunkTL.set(WordFactory.nullPointer()); @@ -232,6 +214,12 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + public void tearDown() { + buffer.release(); + workers.forEach(PlatformThreads::exit); + } + private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } From b849c4a51c499110bf3820337d17c548b919ad37 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 7 Dec 2022 10:17:32 +0300 Subject: [PATCH 76/99] Thread safe Reference handling --- .../genscavenge/ReferenceObjectProcessing.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index e0bffb409384..4021ecf30644 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceInternals; import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; @@ -48,7 +49,7 @@ /** Discovers and handles {@link Reference} objects during garbage collection. */ final class ReferenceObjectProcessing { /** Head of the linked list of discovered references that need to be revisited. */ - private static Reference rememberedRefsList; + private static final AtomicReference> rememberedRefsList = new AtomicReference<>(); /** * For a {@link SoftReference}, the longest duration after its last access to keep its referent @@ -141,9 +142,11 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { // are revisited after the GC finished promoting all strongly reachable objects. // null link means undiscovered, avoid for the last node with a cyclic reference - Reference next = (rememberedRefsList != null) ? rememberedRefsList : dr; - ReferenceInternals.setNextDiscovered(dr, next); - rememberedRefsList = dr; + Reference expected; + do { + expected = rememberedRefsList.get(); + ReferenceInternals.setNextDiscovered(dr, expected != null ? expected : dr); + } while (!rememberedRefsList.compareAndSet(expected, dr)); } /** @@ -154,8 +157,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { */ static Reference processRememberedReferences() { Reference pendingHead = null; - Reference current = rememberedRefsList; - rememberedRefsList = null; + Reference current = rememberedRefsList.getAndSet(null); while (current != null) { // Get the next node (the last node has a cyclic reference to self). @@ -180,7 +182,7 @@ static Reference processRememberedReferences() { } static void afterCollection(UnsignedWord freeBytes) { - assert rememberedRefsList == null; + assert rememberedRefsList.get() == null; UnsignedWord unused = freeBytes.unsignedDivide(1024 * 1024 /* MB */); maxSoftRefAccessIntervalMs = unused.multiply(SerialGCOptions.SoftRefLRUPolicyMSPerMB.getValue()); ReferenceInternals.updateSoftReferenceClock(); From 14c2047b801262e84f784e03c8406b7d73d9cd21 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 7 Dec 2022 12:39:02 +0300 Subject: [PATCH 77/99] Made SerialGC options work with ParallelGC --- .../core/genscavenge/JfrGCEventSupport.java | 2 +- .../SerialAndEpsilonGCOptions.java | 2 +- .../svm/core/genscavenge/SerialGCOptions.java | 38 +++++++++---------- .../UseMarkAndCopyOrEpsilonGC.java | 2 +- .../genscavenge/graal/BarrierSnippets.java | 2 +- .../graal/GenScavengeGCFeature.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 4 +- .../heap/Target_java_lang_ref_Reference.java | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 5270d0282244..af3c72b7c4a3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -141,7 +141,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useSerialOrParallelGC(); + return SubstrateOptions.useMarkAndCopyGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index e6b76ab52316..2985036e6a48 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -80,7 +80,7 @@ private SerialAndEpsilonGCOptions() { } private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseMarkAndCopyOrEpsilonGC()) { + if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index d5e6151e0f54..7bed2ef84982 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -40,16 +40,16 @@ /** Options that are only valid for the serial GC (and not for the epsilon GC). */ public final class SerialGCOptions { @Option(help = "The garbage collection policy, either Adaptive (default) or BySpaceAndTime. Serial GC only.", type = OptionType.User)// - public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::serialGCOnly); + public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::markAndCopyGCOnly); @Option(help = "Percentage of total collection time that should be spent on young generation collections. Serial GC with collection policy 'BySpaceAndTime' only.", type = OptionType.User)// - public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::markAndCopyGCOnly); @Option(help = "The maximum free bytes reserved for allocations, in bytes (0 for automatic according to GC policy). Serial GC only.", type = OptionType.User)// - public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Maximum number of survivor spaces. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly) { + public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly) { @Override public Integer getValueOrDefault(UnmodifiableEconomicMap, Object> values) { Integer value = (Integer) values.get(this); @@ -65,50 +65,50 @@ public Integer getValue(OptionValues values) { }; @Option(help = "Determines if a full GC collects the young generation separately or together with the old generation. Serial GC only.", type = OptionType.Expert) // - public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Enables card marking for image heap objects, which arranges them in chunks. Automatically enabled when supported. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "This number of milliseconds multiplied by the free heap memory in MByte is the time span " + "for which a soft reference will keep its referent alive after its last access. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the shape of the heap before and after each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print summary GC information after application main method returns. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print a time stamp at each collection, if +PrintGC or +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the time for each of the phases of each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Instrument write barriers with counters. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify the remembered set if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify all object references if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify write barriers. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Trace heap chunks during collections, if +VerboseGC and +PrintHeapShape. Serial GC only.", type = OptionType.Debug) // - public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Develop demographics of the object references visited. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); private SerialGCOptions() { } - private static void serialGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue()) { + private static void markAndCopyGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useMarkAndCopyGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java index 9e824c594140..7ab0a5f6d284 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java @@ -35,6 +35,6 @@ public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); + return SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 69adc25a7743..56a2b518cb2c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -204,7 +204,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useSerialOrParallelGC() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useMarkAndCopyGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index e9efa8939ccf..4959d7eb3845 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -156,7 +156,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.useSerialOrParallelGC()) { + if (SubstrateOptions.useMarkAndCopyGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 556b2fe13a46..610237996463 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -350,12 +350,12 @@ private static void validateParallelGC(HostedOptionKey optionKey, boole } @Fold - public static boolean useSerialOrParallelGC() { + public static boolean useMarkAndCopyGC() { return UseSerialGC.getValue() || UseParallelGC.getValue(); } @Fold - public static boolean UseMarkAndCopyOrEpsilonGC() { + public static boolean useMarkAndCopyOrEpsilonGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index 2a3fdd519d37..557818613de9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -232,6 +232,6 @@ public Object transform(Object receiver, Object originalValue) { class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); + return !SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } From 271134d8f456892965562f3707ce6324d03bda6a Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 8 Dec 2022 13:33:26 +0300 Subject: [PATCH 78/99] Enabled incremental collections --- .../oracle/svm/core/genscavenge/GCImpl.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 57 +++++++++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 16 +++--- .../com/oracle/svm/core/SubstrateOptions.java | 9 +-- .../oracle/svm/core/hub/LayoutEncoding.java | 1 - 5 files changed, 52 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dbc9ea720977..41382ef50abd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1042,10 +1042,10 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (ParallelGC.isEnabled()) { - ParallelGCImpl.waitForIdle(); - } else if (isIncremental) { + if (isIncremental) { scanGreyObjectsLoop(); + } else if (ParallelGC.isEnabled()) { + ParallelGCImpl.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index b5d09eaabe72..4c7798de2554 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -167,14 +167,27 @@ public Log report(Log log, boolean traceHeapChunks) { */ @AlwaysInline("GC performance") private Pointer allocateMemory(UnsignedWord objectSize) { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + return allocateMemoryParallel(objectSize); + } Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk; - if (ParallelGC.isEnabled()) { - oldChunk = ParallelGCImpl.getThreadLocalChunk(); - } else { - oldChunk = getLastAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } + if (result.isNonNull()) { + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ + return allocateInNewChunk(objectSize); + } + + @AlwaysInline("GC performance") + private Pointer allocateMemoryParallel(UnsignedWord objectSize) { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the thread local allocation chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -182,7 +195,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(oldChunk, objectSize); + return allocateInNewChunkParallel(oldChunk, objectSize); } /** @@ -191,24 +204,32 @@ private Pointer allocateMemory(UnsignedWord objectSize) { @AlwaysInline("GC performance") private Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } - private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { - if (ParallelGC.isEnabled()) { - ParallelGCImpl.mutex.lock(); + private Pointer allocateInNewChunk(UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + if (newChunk.isNonNull()) { + return AlignedHeapChunk.allocateMemory(newChunk, objectSize); + } + return WordFactory.nullPointer(); + } + + private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk; + ParallelGCImpl.mutex.lock(); + try { if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } - } - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - if (ParallelGC.isEnabled()) { + newChunk = requestAlignedHeapChunk(); + } finally { ParallelGCImpl.mutex.unlock(); - ParallelGCImpl.setThreadLocalChunk(newChunk); } if (newChunk.isNonNull()) { + ParallelGCImpl.setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); @@ -379,7 +400,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - return promoteAlignedObjectParallel(original, originalSpace); + return copyAlignedObjectParallel(original); } Object copy = copyAlignedObject(original); @@ -390,7 +411,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") - Object promoteAlignedObjectParallel(Object original, Space originalSpace) { + Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); @@ -485,7 +506,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); @@ -511,7 +532,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index dd1668e2353e..24ad775b1e91 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -34,7 +34,6 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; -import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; @@ -88,23 +87,21 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static void waitForIdle() { - singleton().waitForIdleImpl(); - } - public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { return scannedChunkTL.get(); } - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert chunk.isNonNull(); allocChunkTL.set(chunk); } public void push(Pointer ptr) { + assert ptr.isNonNull(); buffer.push(ptr); if (inParallelPhase) { parPhase.signal(); @@ -168,7 +165,10 @@ private Thread startWorkerThread(int n) { return t; } - private void waitForIdleImpl() { + /** + * Start parallel phase and wait until all chunks have been processed. Used by complete collections. + */ + public void waitForIdle() { assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); allocChunkTL.set(WordFactory.nullPointer()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 610237996463..2fae63ed226e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -339,13 +339,8 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o }; private static void validateParallelGC(OptionKey unused) { - validateParallelGC(SubstrateOptions.MultiThreaded, true); - validateParallelGC(ConcealedOptions.UseRememberedSet, false); - } - - private static void validateParallelGC(HostedOptionKey optionKey, boolean expected) { - if (optionKey.getValue() != expected) { - throw UserError.abort("ParallelGC requires the option %s to be %s", optionKey, expected); + if (!SubstrateOptions.MultiThreaded.getValue()) { + throw new InterruptImageBuilding("ParallelGC requires the option MultiThreaded to be set."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index b290969c630a..a60463fdcdbb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -274,7 +274,6 @@ public static UnsignedWord getSizeFromObjectInline(Object obj) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { - assert encoding == KnownIntrinsics.readHub(obj).getLayoutEncoding(); if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From bced7d1fc28e09f9a130b867b5f6f6b299414a42 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 12 Dec 2022 13:23:17 +0300 Subject: [PATCH 79/99] Use try-finally for mutex locking --- .../oracle/svm/core/genscavenge/Space.java | 30 +++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 43 +++++++++++-------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 4c7798de2554..828c65e92c49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -504,12 +504,15 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + try { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + } finally { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } } @@ -530,12 +533,15 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + try { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + } finally { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 24ad775b1e91..07f7cac21fed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,7 +53,6 @@ import org.graalvm.word.WordFactory; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -61,7 +60,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; private List workers; - private final AtomicInteger busyWorkers = new AtomicInteger(); + private int busyWorkers; /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -112,7 +111,7 @@ public void push(Pointer ptr) { public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); - busyWorkers.set(workerCount); + busyWorkers = workerCount; workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); } @@ -138,16 +137,19 @@ private Thread startWorkerThread(int n) { try { Pointer ptr; while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { - if (busyWorkers.decrementAndGet() == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); mutex.lock(); - parPhase.block(); - mutex.unlock(); - busyWorkers.incrementAndGet(); - debugLog().string("WW run ").unsigned(n).newline(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); + parPhase.block(); + ++busyWorkers; + debugLog().string("WW run ").unsigned(n).newline(); + } finally { + mutex.unlock(); + } } do { @@ -174,15 +176,18 @@ public void waitForIdle() { allocChunkTL.set(WordFactory.nullPointer()); debugLog().string("PP start workers\n"); - inParallelPhase = true; - parPhase.broadcast(); // let worker threads run - mutex.lock(); - while (inParallelPhase) { - debugLog().string("PP wait\n"); - seqPhase.block(); // wait for them to become idle + try { + inParallelPhase = true; + parPhase.broadcast(); // let worker threads run + + while (inParallelPhase) { + debugLog().string("PP wait\n"); + seqPhase.block(); // wait for them to become idle + } + } finally { + mutex.unlock(); } - mutex.unlock(); } private void scanChunk(Pointer ptr) { From 165a8dcca7d99f0900c81001825504c3aa3c99fc Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 13 Dec 2022 15:08:19 +0300 Subject: [PATCH 80/99] Added assertion to `ParallelGCImpl.getScannedChunk()` --- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 828c65e92c49..682a32ed4c72 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -221,7 +221,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu AlignedHeapChunk.AlignedHeader newChunk; ParallelGCImpl.mutex.lock(); try { - if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { + if (oldChunk.notEqual(ParallelGCImpl.getScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } newChunk = requestAlignedHeapChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 07f7cac21fed..2a0c9af3c30b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -86,7 +86,8 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { + public static AlignedHeapChunk.AlignedHeader getScannedChunk() { + assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); return scannedChunkTL.get(); } From 8ec4ce900c7f4d5314900f6f5a0ce916a65adcb4 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 13 Dec 2022 15:44:55 +0300 Subject: [PATCH 81/99] Wait for workers to be blocked before starting parallel phase --- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 2a0c9af3c30b..ed0b2bbbefd8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -176,15 +176,20 @@ public void waitForIdle() { push(HeapChunk.asPointer(allocChunkTL.get())); allocChunkTL.set(WordFactory.nullPointer()); - debugLog().string("PP start workers\n"); mutex.lock(); try { + while (busyWorkers > 0) { // wait for worker threads to become ready + debugLog().string("PP wait for workers\n"); + seqPhase.block(); + } + + debugLog().string("PP start workers\n"); inParallelPhase = true; - parPhase.broadcast(); // let worker threads run + parPhase.broadcast(); // let worker threads run while (inParallelPhase) { debugLog().string("PP wait\n"); - seqPhase.block(); // wait for them to become idle + seqPhase.block(); // wait for them to become idle } } finally { mutex.unlock(); From e4ec854b206013117445e4776a9b90fdd18a4699 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 26 Jan 2023 10:49:16 +0300 Subject: [PATCH 82/99] Don't retire scanned allocation chunks Instead, remember top position and rescan starting from it if needed --- .../oracle/svm/core/genscavenge/Space.java | 4 +- .../genscavenge/parallel/ParallelGCImpl.java | 64 +++++++++++++------ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 682a32ed4c72..a0838fb9cb36 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -221,9 +221,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu AlignedHeapChunk.AlignedHeader newChunk; ParallelGCImpl.mutex.lock(); try { - if (oldChunk.notEqual(ParallelGCImpl.getScannedChunk())) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); - } + ParallelGCImpl.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index ed0b2bbbefd8..91cb59ab317c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -50,6 +50,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import java.util.List; @@ -59,7 +60,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private List workers; + private Thread[] workers; private int busyWorkers; /** @@ -69,6 +70,8 @@ public class ParallelGCImpl extends ParallelGC { FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); + private static final FastThreadLocalWord allocChunkScanOffsetTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -86,11 +89,6 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static AlignedHeapChunk.AlignedHeader getScannedChunk() { - assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); - return scannedChunkTL.get(); - } - public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } @@ -98,6 +96,7 @@ public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); allocChunkTL.set(chunk); + allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); } public void push(Pointer ptr) { @@ -108,12 +107,23 @@ public void push(Pointer ptr) { } } + public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + if (chunk.notEqual(scannedChunkTL.get())) { + UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + assert scanOffset.aboveThan(0); + if (chunk.getTopOffset().aboveThan(scanOffset)) { + push(HeapChunk.asPointer(chunk).add(scanOffset)); + } + } + } + @Override public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); busyWorkers = workerCount; - workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); + workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); } private int getWorkerCount() { @@ -137,7 +147,7 @@ private Thread startWorkerThread(int n) { while (true) { try { Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { mutex.lock(); try { if (--busyWorkers == 0) { @@ -174,7 +184,6 @@ private Thread startWorkerThread(int n) { public void waitForIdle() { assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); - allocChunkTL.set(WordFactory.nullPointer()); mutex.lock(); try { @@ -194,6 +203,11 @@ public void waitForIdle() { } finally { mutex.unlock(); } + // clean up thread local allocation chunks + allocChunkTL.set(WordFactory.nullPointer()); + for (Thread t: workers) { + allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); + } } private void scanChunk(Pointer ptr) { @@ -202,23 +216,35 @@ private void scanChunk(Pointer ptr) { if (ptr.and(UNALIGNED_BIT).notEqual(0)) { UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); + if (chunk.equal(ptr)) { + ptr = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); + } + HeapChunk.walkObjectsFromInline(chunk, ptr, getVisitor()); } } } private void scanAllocChunk() { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - if (allocChunk.isNonNull()) { - debugLog().string("WW scan alloc=").zhex(allocChunk).newline(); + if (allocChunkNeedsScanning()) { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + assert scanOffset.aboveThan(0); + Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); + debugLog().string("WW scan alloc=").zhex(allocChunk).string(" from offset ").unsigned(scanOffset).newline(); scannedChunkTL.set(allocChunk); - AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, getVisitor()); + scannedChunkTL.set(WordFactory.nullPointer()); if (allocChunkTL.get().equal(allocChunk)) { - // this allocation chunk is now black, retire it - allocChunkTL.set(WordFactory.nullPointer()); + // remember top offset so that we don't scan the same objects again + allocChunkScanOffsetTL.set(allocChunk.getTopOffset()); } } - scannedChunkTL.set(WordFactory.nullPointer()); + } + + private boolean allocChunkNeedsScanning() { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } private GreyToBlackObjectVisitor getVisitor() { @@ -228,7 +254,9 @@ private GreyToBlackObjectVisitor getVisitor() { @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { buffer.release(); - workers.forEach(PlatformThreads::exit); + for (Thread t: workers) { + PlatformThreads.exit(t); + } } private static Log verboseGCLog() { From dd322190026be28d615aa12344ff1e5220ac1ee8 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 14:12:45 +0300 Subject: [PATCH 83/99] Merged ParallelGC and ParallelGCImpl --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +++-- .../oracle/svm/core/genscavenge/HeapImpl.java | 10 ++-- .../oracle/svm/core/genscavenge/Space.java | 45 +++++++++--------- .../genscavenge/parallel/ChunkBuffer.java | 6 +-- .../{ParallelGCImpl.java => ParallelGC.java} | 21 +++++---- .../graal/snippets/CEntryPointSnippets.java | 6 +-- .../src/com/oracle/svm/core/heap/Heap.java | 2 + .../com/oracle/svm/core/heap/ParallelGC.java | 47 ------------------- 8 files changed, 57 insertions(+), 91 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/{ParallelGCImpl.java => ParallelGC.java} (95%) delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 98ac4586a023..090d269a271e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -64,7 +64,7 @@ import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.graal.RuntimeCompilation; import com.oracle.svm.core.heap.CodeReferenceMapDecoder; @@ -72,7 +72,6 @@ import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -146,6 +145,12 @@ public void collect(GCCause cause) { collect(cause, false); } + public void initialize() { + if (ParallelGC.isEnabled()) { + ParallelGC.singleton().startWorkerThreads(); + } + } + public void maybeCollectOnAllocation() { boolean outOfMemory = false; if (hasNeverCollectPolicy()) { @@ -1055,7 +1060,7 @@ private void scanGreyObjects(boolean isIncremental) { if (isIncremental) { scanGreyObjectsLoop(); } else if (ParallelGC.isEnabled()) { - ParallelGCImpl.singleton().waitForIdle(); + ParallelGC.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index c73583c9d8ac..2eadb6f75e1b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -49,7 +49,6 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; @@ -59,7 +58,7 @@ import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -204,7 +203,7 @@ boolean walkMemory(MemoryWalker.Visitor visitor) { @Uninterruptible(reason = "Tear-down in progress.") public boolean tearDown() { if (ParallelGC.isEnabled()) { - ParallelGCImpl.singleton().tearDown(); + ParallelGC.singleton().tearDown(); } youngGeneration.tearDown(); oldGeneration.tearDown(); @@ -243,6 +242,11 @@ GCImpl getGCImpl() { return gcImpl; } + @Override + public void initGC() { + gcImpl.initialize(); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 26201ad5a6d3..ec067cfbe591 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -44,10 +44,9 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -191,7 +190,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -207,8 +206,8 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer retractAllocation(UnsignedWord objectSize) { - assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } @@ -223,15 +222,15 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk; - ParallelGCImpl.mutex.lock(); + ParallelGC.mutex.lock(); try { - ParallelGCImpl.singleton().pushAllocChunk(oldChunk); + ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { - ParallelGCImpl.mutex.unlock(); + ParallelGC.mutex.unlock(); } if (newChunk.isNonNull()) { - ParallelGCImpl.setAllocationChunk(newChunk); + ParallelGC.setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); @@ -401,7 +400,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { return copyAlignedObjectParallel(original); } @@ -415,7 +414,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); - assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -522,17 +521,17 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.lock(); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lock(); } try { originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); } finally { - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlock(); } } } @@ -551,17 +550,17 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.lock(); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lock(); } try { originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); } finally { - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlock(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 60a1614ded48..17dbc628ef08 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -54,7 +54,7 @@ static int wordSize() { } void push(Pointer ptr) { - assert !ParallelGCImpl.isInParallelPhase() || ParallelGCImpl.mutex.isOwner(); + assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.isOwner(); if (top >= size) { int oldSize = size; size *= 2; @@ -66,7 +66,7 @@ void push(Pointer ptr) { } Pointer pop() { - ParallelGCImpl.mutex.lock(); + ParallelGC.mutex.lock(); try { if (top > 0) { top -= wordSize(); @@ -75,7 +75,7 @@ Pointer pop() { return WordFactory.nullPointer(); } } finally { - ParallelGCImpl.mutex.unlock(); + ParallelGC.mutex.unlock(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java similarity index 95% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 91cb59ab317c..5de72d3949ae 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -27,6 +27,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; @@ -35,7 +36,6 @@ import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; @@ -53,10 +53,9 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import java.util.List; import java.util.stream.IntStream; -public class ParallelGCImpl extends ParallelGC { +public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; @@ -81,8 +80,13 @@ public class ParallelGCImpl extends ParallelGC { private volatile boolean inParallelPhase; @Fold - public static ParallelGCImpl singleton() { - return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + public static ParallelGC singleton() { + return ImageSingletons.lookup(ParallelGC.class); + } + + @Fold + public static boolean isEnabled() { + return SubstrateOptions.UseParallelGC.getValue(); } public static boolean isInParallelPhase() { @@ -108,7 +112,7 @@ public void push(Pointer ptr) { } public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + assert ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); @@ -118,8 +122,7 @@ public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { } } - @Override - public void startWorkerThreadsImpl() { + public void startWorkerThreads() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); busyWorkers = workerCount; @@ -280,6 +283,6 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { @Override public void afterRegistration(AfterRegistrationAccess access) { - ImageSingletons.add(ParallelGC.class, new ParallelGCImpl()); + ImageSingletons.add(ParallelGC.class, new ParallelGC()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 9514d24d7248..e46003720772 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -86,7 +86,7 @@ import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode; import com.oracle.svm.core.graal.nodes.CEntryPointLeaveNode; import com.oracle.svm.core.graal.nodes.CEntryPointUtilityNode; -import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -325,8 +325,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - /* Number of workers can be affected by a runtime option, so start threads after options are parsed */ - ParallelGC.startWorkerThreads(); + /* Number of parallel GC workers can be affected by a runtime option, so call this after options are parsed */ + Heap.getHeap().initGC(); boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index 4902b9c13dd5..99fa80d694c0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -80,6 +80,8 @@ protected Heap() { public abstract GC getGC(); + public abstract void initGC(); + public abstract RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport(); /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java deleted file mode 100644 index 95d8de510c9f..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.heap; - -import com.oracle.svm.core.SubstrateOptions; -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; - -public abstract class ParallelGC { - - @Fold - public static boolean isEnabled() { - return SubstrateOptions.UseParallelGC.getValue(); - } - - public static void startWorkerThreads() { - if (isEnabled()) { - ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); - } - } - - public abstract void startWorkerThreadsImpl(); -} From d9f38f327dd83b3a6d45dcd8a96cacfff85c102f Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 15:47:48 +0300 Subject: [PATCH 84/99] Introduced special SafepointBehavior for worker threads --- .../svm/core/genscavenge/parallel/ParallelGC.java | 2 +- .../src/com/oracle/svm/core/thread/Safepoint.java | 6 ++++-- .../src/com/oracle/svm/core/thread/VMThreads.java | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 5de72d3949ae..299ae2e88a1c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -144,7 +144,7 @@ private int getDefaultWorkerCount() { private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.markThreadAsCrashed(); + VMThreads.SafepointBehavior.useAsParallelGCThread(); debugLog().string("WW start ").unsigned(n).newline(); while (true) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 01f45420aa5f..7f4c0ea82a20 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -760,7 +760,8 @@ private static void waitForSafepoints(String reason) { int safepointBehavior = SafepointBehavior.getSafepointBehaviorVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -905,7 +906,8 @@ public static int countingVMOperation() { int status = StatusSupport.getStatusVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 1c2df2c689fc..c0b1bc3502df 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -865,6 +865,9 @@ public static class SafepointBehavior { */ static final int THREAD_CRASHED = 2; + /** Similar to THREAD_CRASHED, used by ParallelGC worker threads. */ + static final int PARALLEL_GC_THREAD = 3; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean ignoresSafepoints() { return safepointBehaviorTL.getVolatile() != ALLOW_SAFEPOINT; @@ -915,6 +918,14 @@ public static void markThreadAsCrashed() { safepointBehaviorTL.setVolatile(THREAD_CRASHED); } + /** + * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, + * so it must not keep references to movable heap objects on its stack. + */ + public static void useAsParallelGCThread() { + safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static String toString(int safepointBehavior) { switch (safepointBehavior) { @@ -924,6 +935,8 @@ public static String toString(int safepointBehavior) { return "PREVENT_VM_FROM_REACHING_SAFEPOINT"; case THREAD_CRASHED: return "THREAD_CRASHED"; + case PARALLEL_GC_THREAD: + return "PARALLEL_GC_THREAD"; default: return "Invalid safepoint behavior"; } From ec3b2a59dd3feee4d13d1e8575a9eba5e1daac1c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 15:48:33 +0300 Subject: [PATCH 85/99] Style fixes --- .../oracle/svm/core/genscavenge/GCImpl.java | 1 + .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../oracle/svm/core/genscavenge/Space.java | 1 + .../genscavenge/parallel/ChunkBuffer.java | 15 +++++++------- .../core/genscavenge/parallel/ParallelGC.java | 20 +++++++++++-------- .../com/oracle/svm/core/SubstrateOptions.java | 8 ++++---- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 090d269a271e..c6e5b1f337db 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -145,6 +145,7 @@ public void collect(GCCause cause) { collect(cause, false); } + @SuppressWarnings("static-method") public void initialize() { if (ParallelGC.isEnabled()) { ParallelGC.singleton().startWorkerThreads(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 1b35a0807d4f..7778ba7e46cc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -33,7 +33,6 @@ import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ec067cfbe591..185e885ad33a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -205,6 +205,7 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") + @SuppressWarnings("static-method") private Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 17dbc628ef08..6a6fdc06aa84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -41,7 +41,8 @@ public class ChunkBuffer { private static final int INITIAL_SIZE = 1024 * wordSize(); private Pointer buffer; - private int size, top; + private int size; + private int top; @Fold static int wordSize() { @@ -84,12 +85,12 @@ void release() { } @AlwaysInline("GC performance") - private Pointer malloc(int size) { - return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(size)); + private static Pointer malloc(int bytes) { + return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(bytes)); } @AlwaysInline("GC performance") - private Pointer realloc(Pointer orig, int origSize, int newSize) { + private static Pointer realloc(Pointer orig, int origSize, int newSize) { Pointer ptr = malloc(newSize); UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); free(orig, origSize); @@ -97,7 +98,7 @@ private Pointer realloc(Pointer orig, int origSize, int newSize) { } @AlwaysInline("GC performance") - private void free(Pointer ptr, int size) { - CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(size)); + private static void free(Pointer ptr, int bytes) { + CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(bytes)); } -} \ No newline at end of file +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 299ae2e88a1c..f57ff42c0f3a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -129,14 +129,14 @@ public void startWorkerThreads() { workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); } - private int getWorkerCount() { + private static int getWorkerCount() { int setting = ParallelGCOptions.ParallelGCThreads.getValue(); - int workers = setting > 0 ? setting : getDefaultWorkerCount(); - verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); - return workers; + int count = setting > 0 ? setting : getDefaultWorkerCount(); + verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(count).string("]").newline(); + return count; } - private int getDefaultWorkerCount() { + private static int getDefaultWorkerCount() { // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() int cpus = Jvm.JVM_ActiveProcessorCount(); return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; @@ -213,21 +213,24 @@ public void waitForIdle() { } } + @SuppressWarnings("static-method") private void scanChunk(Pointer ptr) { if (ptr.isNonNull()) { debugLog().string("WW scan chunk=").zhex(ptr).newline(); if (ptr.and(UNALIGNED_BIT).notEqual(0)) { UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { + Pointer start = ptr; AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); if (chunk.equal(ptr)) { - ptr = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); + start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); } - HeapChunk.walkObjectsFromInline(chunk, ptr, getVisitor()); + HeapChunk.walkObjectsFromInline(chunk, start, getVisitor()); } } } + @SuppressWarnings("static-method") private void scanAllocChunk() { if (allocChunkNeedsScanning()) { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); @@ -245,12 +248,13 @@ private void scanAllocChunk() { } } + @SuppressWarnings("static-method") private boolean allocChunkNeedsScanning() { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } - private GreyToBlackObjectVisitor getVisitor() { + private static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 10f0049ca1c4..af819661c0de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -36,7 +36,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Set; import java.util.function.Predicate; import com.oracle.svm.core.util.InterruptImageBuilding; @@ -317,7 +316,7 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// @Option(help = "Use a parallel GC")// - public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::validateParallelGC) { + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::requireMultiThreading) { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { @@ -327,9 +326,10 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; - private static void validateParallelGC(OptionKey unused) { + private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { - throw new InterruptImageBuilding("ParallelGC requires the option MultiThreaded to be set."); + throw new InterruptImageBuilding( + "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); } } From 1462c2f124f4443c362cf3863ad2019e1e191cf4 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Mar 2023 15:04:16 +0300 Subject: [PATCH 86/99] Fall back to serial GC if collection occurs before worker threads have been started --- .../oracle/svm/core/genscavenge/GCImpl.java | 8 +++-- .../genscavenge/parallel/ChunkBuffer.java | 6 ++++ .../core/genscavenge/parallel/ParallelGC.java | 36 +++++++++++++------ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c6e5b1f337db..eee115fc8d88 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1060,8 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) { try { if (isIncremental) { scanGreyObjectsLoop(); - } else if (ParallelGC.isEnabled()) { - ParallelGC.singleton().waitForIdle(); + } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { + // GC can happen before parallel worker threads have been started. In this case, fall back to serial GC. } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } @@ -1159,6 +1159,10 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + + if (ParallelGC.isEnabled()) { + ParallelGC.singleton().cleanupAfterCollection(); + } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 6a6fdc06aa84..757e3e02c324 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -67,6 +67,7 @@ void push(Pointer ptr) { } Pointer pop() { + assert ParallelGC.isInParallelPhase(); ParallelGC.mutex.lock(); try { if (top > 0) { @@ -80,6 +81,11 @@ Pointer pop() { } } + boolean isEmpty() { + assert !ParallelGC.isInParallelPhase(); + return top == 0; + } + void release() { free(buffer, size); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index f57ff42c0f3a..86affe89570a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -59,9 +59,6 @@ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private Thread[] workers; - private int busyWorkers; - /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -77,6 +74,8 @@ public class ParallelGC { private final VMCondition parPhase = new VMCondition(mutex); private ChunkBuffer buffer; + private Thread[] workers; + private int busyWorkers; private volatile boolean inParallelPhase; @Fold @@ -105,14 +104,16 @@ public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { public void push(Pointer ptr) { assert ptr.isNonNull(); - buffer.push(ptr); - if (inParallelPhase) { - parPhase.signal(); + if (buffer != null) { + buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); + } } } public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); @@ -182,9 +183,15 @@ private Thread startWorkerThread(int n) { } /** - * Start parallel phase and wait until all chunks have been processed. Used by complete collections. + * Start parallel phase and wait until all chunks have been processed. + * @return false if worker threads have not been started yet. This can happen if GC happens very early + * during application startup. */ - public void waitForIdle() { + public boolean waitForIdle() { + if (workers == null) { + return false; + } + assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); @@ -206,11 +213,13 @@ public void waitForIdle() { } finally { mutex.unlock(); } + + assert buffer.isEmpty(); // clean up thread local allocation chunks - allocChunkTL.set(WordFactory.nullPointer()); for (Thread t: workers) { allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); } + return true; } @SuppressWarnings("static-method") @@ -258,10 +267,15 @@ private static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + @SuppressWarnings("static-method") + public void cleanupAfterCollection() { + allocChunkTL.set(WordFactory.nullPointer()); + } + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { buffer.release(); - for (Thread t: workers) { + for (Thread t : workers) { PlatformThreads.exit(t); } } From df092558bdc5f0428eab25a094c9a96448afd7ca Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 14 Mar 2023 13:08:56 +0300 Subject: [PATCH 87/99] Style fixes --- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +- .../SerialAndEpsilonGCOptions.java | 3 +- .../oracle/svm/core/genscavenge/Space.java | 3 +- .../core/genscavenge/parallel/ParallelGC.java | 71 +++++++++---------- .../parallel/ParallelGCOptions.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 2 +- .../graal/snippets/CEntryPointSnippets.java | 3 +- .../com/oracle/svm/core/thread/Safepoint.java | 4 +- .../com/oracle/svm/core/thread/VMThreads.java | 8 +-- 9 files changed, 51 insertions(+), 48 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index eee115fc8d88..5e4249786a0a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1061,7 +1061,8 @@ private void scanGreyObjects(boolean isIncremental) { if (isIncremental) { scanGreyObjectsLoop(); } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { - // GC can happen before parallel worker threads have been started. In this case, fall back to serial GC. + // GC can happen before parallel worker threads have been started. In this case, + // fall back to serial GC. } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index 8cc7d3229f42..f1f7d372a8a0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -81,7 +81,8 @@ private SerialAndEpsilonGCOptions() { private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); + "The option '" + optionKey.getName() + + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 185e885ad33a..b744cae07cea 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -442,7 +442,8 @@ Object copyAlignedObjectParallel(Object original) { // Install forwarding pointer into the original header Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header + // We have won the race, now we must copy the object bits. First install the original + // header copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object if (hubOffset > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 86affe89570a..ff25c67e4eff 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -60,14 +60,12 @@ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; /** - * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. + * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new + * chunk needs to be allocated. */ - private static final FastThreadLocalWord allocChunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); - private static final FastThreadLocalWord scannedChunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); - private static final FastThreadLocalWord allocChunkScanOffsetTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); + private static final FastThreadLocalWord allocChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); + private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); + private static final FastThreadLocalWord allocChunkScanOffsetTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -145,36 +143,36 @@ private static int getDefaultWorkerCount() { private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.useAsParallelGCThread(); - debugLog().string("WW start ").unsigned(n).newline(); - - while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lock(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); - parPhase.block(); - ++busyWorkers; - debugLog().string("WW run ").unsigned(n).newline(); - } finally { - mutex.unlock(); + VMThreads.SafepointBehavior.useAsParallelGCThread(); + debugLog().string("WW start ").unsigned(n).newline(); + + while (true) { + try { + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { + mutex.lock(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); } + debugLog().string("WW idle ").unsigned(n).newline(); + parPhase.block(); + ++busyWorkers; + debugLog().string("WW run ").unsigned(n).newline(); + } finally { + mutex.unlock(); } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); } + } }); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); @@ -184,8 +182,9 @@ private Thread startWorkerThread(int n) { /** * Start parallel phase and wait until all chunks have been processed. - * @return false if worker threads have not been started yet. This can happen if GC happens very early - * during application startup. + * + * @return false if worker threads have not been started yet. This can happen if GC happens very + * early during application startup. */ public boolean waitForIdle() { if (workers == null) { @@ -216,7 +215,7 @@ public boolean waitForIdle() { assert buffer.isEmpty(); // clean up thread local allocation chunks - for (Thread t: workers) { + for (Thread t : workers) { allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); } return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 6fe4b04d6ae9..3857cd35abc2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -34,6 +34,6 @@ public class ParallelGCOptions { - @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index af819661c0de..8b4e3967cec3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -329,7 +329,7 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); + "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index e46003720772..c8506b69a77e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -325,7 +325,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - /* Number of parallel GC workers can be affected by a runtime option, so call this after options are parsed */ + // Number of parallel GC workers can be affected by a runtime option, + // so call this after options are parsed Heap.getHeap().initGC(); boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 7f4c0ea82a20..40591ee0e12e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -761,7 +761,7 @@ private static void waitForSafepoints(String reason) { if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -907,7 +907,7 @@ public static int countingVMOperation() { if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index c0b1bc3502df..f89f63b671e7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -887,7 +887,7 @@ public static int getSafepointBehaviorVolatile(IsolateThread vmThread) { * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * thread will also actively prevent the VM from reaching a safepoint (regardless of its * thread status). - * + * * NOTE: Be careful with this method and make sure that this thread does not allocate any * Java objects as this could result deadlocks. This method will only work prevent * safepoints reliably if it is called from a thread with @@ -902,7 +902,7 @@ public static void preventSafepoints() { /** * Marks the thread as crashed. This method may only be used in places where it is not * possible to safely detach a thread. - * + * * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * safepoint handling will ignore the thread so that the VM can reach a safepoint regardless * of the status of this thread. @@ -919,8 +919,8 @@ public static void markThreadAsCrashed() { } /** - * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, - * so it must not keep references to movable heap objects on its stack. + * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so + * it must not keep references to movable heap objects on its stack. */ public static void useAsParallelGCThread() { safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); From 34ad44de4a77eabf28c3537ed9a4a03b0f9021d6 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 13 Mar 2023 16:50:45 +0100 Subject: [PATCH 88/99] Fix after rebasing to master. --- .../src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 167323c0ea64..4bf74c62a9f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -450,7 +450,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); + forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); From 35325d64e9dc63ca8e4d44ffd37b2e3d958b321c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 15 Mar 2023 12:40:35 +0100 Subject: [PATCH 89/99] Make the inner-most part of the GC uninterruptible. --- .../genscavenge/AbstractCollectionPolicy.java | 2 + .../core/genscavenge/AlignedHeapChunk.java | 4 + .../genscavenge/BasicCollectionPolicies.java | 3 + .../core/genscavenge/ChunksAccounting.java | 3 + .../core/genscavenge/CollectionPolicy.java | 3 + .../svm/core/genscavenge/GCAccounting.java | 2 + .../oracle/svm/core/genscavenge/GCImpl.java | 4 + .../svm/core/genscavenge/Generation.java | 6 + .../genscavenge/GreyToBlackObjRefVisitor.java | 22 ++++ .../genscavenge/GreyToBlackObjectVisitor.java | 2 + .../svm/core/genscavenge/HeapChunk.java | 4 +- .../core/genscavenge/HeapChunkProvider.java | 29 +---- .../svm/core/genscavenge/HeapParameters.java | 1 + .../core/genscavenge/ObjectHeaderImpl.java | 19 ++- .../svm/core/genscavenge/OldGeneration.java | 8 +- .../ReferenceObjectProcessing.java | 6 + .../oracle/svm/core/genscavenge/Space.java | 29 +++-- .../core/genscavenge/UnalignedHeapChunk.java | 1 + .../svm/core/genscavenge/YoungGeneration.java | 10 +- .../genscavenge/parallel/ChunkBuffer.java | 38 +++--- .../core/genscavenge/parallel/ParallelGC.java | 113 +++++++++--------- .../remset/AlignedChunkRememberedSet.java | 9 ++ .../core/genscavenge/remset/CardTable.java | 6 + .../remset/CardTableBasedRememberedSet.java | 7 ++ .../genscavenge/remset/FirstObjectTable.java | 12 ++ .../genscavenge/remset/NoRememberedSet.java | 8 ++ .../genscavenge/remset/RememberedSet.java | 8 ++ .../remset/UnalignedChunkRememberedSet.java | 6 + .../posix/pthread/PthreadVMLockSupport.java | 1 + .../core/windows/WindowsVMLockSupport.java | 1 + .../oracle/svm/core/config/ObjectLayout.java | 2 + .../heap/InstanceReferenceMapDecoder.java | 4 +- .../oracle/svm/core/heap/ObjectHeader.java | 1 + .../oracle/svm/core/heap/OutOfMemoryUtil.java | 3 + .../svm/core/heap/PodReferenceMapDecoder.java | 7 +- .../svm/core/heap/ReferenceInternals.java | 8 +- .../svm/core/heap/ReferenceMapIndex.java | 3 + .../com/oracle/svm/core/hub/DynamicHub.java | 2 + .../svm/core/hub/InteriorObjRefWalker.java | 7 ++ .../oracle/svm/core/hub/LayoutEncoding.java | 5 + .../svm/core/jdk/UninterruptibleUtils.java | 7 ++ .../locks/SingleThreadedVMLockSupport.java | 1 + .../oracle/svm/core/locks/VMCondition.java | 1 + .../os/AbstractCommittedMemoryProvider.java | 1 + .../com/oracle/svm/core/thread/VMThreads.java | 1 + .../svm/core/util/TypedMemoryReader.java | 5 + 46 files changed, 306 insertions(+), 119 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index bee7b2871e1b..1f8e4b4308fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { assert VMOperation.isGCInProgress() : "use only during GC"; guaranteeSizeParametersInitialized(); @@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index bee8b8cbe57d..1a64269b47af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static public interface AlignedHeader extends HeapChunk.Header { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize); } @@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) { } /** Allocate uninitialized memory within this AlignedHeapChunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { Pointer result = WordFactory.nullPointer(); UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -113,6 +115,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { } /** Retract the latest allocation. Used by parallel collector. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); assert newTop.aboveThan(HeapChunk.asPointer(that)); @@ -137,6 +140,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { } /** Return the offset of an object within the objects part of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) { Pointer objectsStart = getObjectsStart(that); return objectPointer.subtract(objectsStart); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java index ae89d3740679..8aa186b7509c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; @@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { return WordFactory.zero(); } @@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { return 1; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 17c24b576832..99b3f184cb1c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -60,6 +60,7 @@ public void reset() { unalignedChunkBytes = WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getChunkBytes() { return getAlignedChunkBytes().add(getUnalignedChunkBytes()); } @@ -117,10 +118,12 @@ private void noteUnaligned(UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void unnoteUnaligned(UnsignedWord size) { unalignedCount--; unalignedChunkBytes = unalignedChunkBytes.subtract(size); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 1d11025ef0f7..ba738c3df223 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; @@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x * this amount can be used for surviving objects. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnsignedWord getSurvivorSpacesCapacity(); /** The capacity of the young generation, comprising the eden and survivor spaces. */ @@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of * survivor spaces + 1}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getTenuringAge(); /** Called at the beginning of a collection, in the safepoint operation. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 128df06b6087..c006ab6faa54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) { /** Called after an object has been promoted from the young generation to the old generation. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c8859d924d5f..3dd868593ce1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -535,6 +535,7 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } @@ -1085,6 +1086,7 @@ private static void scanGreyObjectsLoop() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); @@ -1118,6 +1120,7 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1234,6 +1237,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } + @Fold public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 35b694891c19..e487a4c7f1d2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,6 +27,8 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -66,6 +68,8 @@ public String getName() { * promotion was done by copying, or {@code null} if there was insufficient capacity in * this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); /** @@ -79,6 +83,8 @@ public String getName() { * was promoted through HeapChunk motion, or {@code null} if there was insufficient * capacity in this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 7778ba7e46cc..cad4c249834e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -30,6 +30,7 @@ import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; @@ -57,12 +58,14 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { return visitObjectReferenceInline(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); @@ -128,16 +131,22 @@ public interface Counters extends AutoCloseable { @Override void close(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteObjRef(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNullReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteForwardedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNonHeapReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteCopiedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); void toLog(); @@ -182,32 +191,39 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { objRef += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { nullReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { forwardedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { nonHeapReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { copiedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { + // TEMP (chaeubl): this counter would break unmodifiedReference += 1L; } @@ -241,26 +257,32 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 9a8b48845d46..bc432b866261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; @@ -55,6 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 69104f59be47..43df19d7b1fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -41,8 +41,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.struct.PinnedObjectField; @@ -177,6 +177,7 @@ public interface Header> extends HeaderPadding { void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); @@ -306,6 +307,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 4df6532b8e51..1aeb21c28ad6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -35,11 +34,11 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; @@ -89,23 +88,15 @@ public UnsignedWord getBytesInUnusedChunks() { return bytesInUnusedAlignedChunks.get(); } - @AlwaysInline("Remove all logging when noopLog is returned by this method") - private static Log log() { - return Log.noopLog(); - } - private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk"); private static final OutOfMemoryError UNALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an unaligned heap chunk"); /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); - log().string("[HeapChunkProvider.produceAlignedChunk chunk size: ").unsigned(chunkSize).newline(); - AlignedHeader result = popUnusedAlignedChunk(); - log().string(" unused chunk: ").zhex(result).newline(); - if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ noteFirstAllocationTime(); @@ -113,7 +104,6 @@ AlignedHeader produceAlignedChunk() { if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } - log().string(" new chunk: ").zhex(result).newline(); AlignedHeapChunk.initialize(result, chunkSize); } @@ -123,8 +113,6 @@ AlignedHeader produceAlignedChunk() { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" result chunk: ").zhex(result).string(" ]").newline(); return result; } @@ -200,13 +188,10 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { if (SubstrateOptions.MultiThreaded.getValue()) { VMThreads.guaranteeOwnsThreadMutex("Should hold the lock when pushing to the global list."); } - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); HeapChunk.setNext(chunk, unusedAlignedChunks.get()); unusedAlignedChunks.set(chunk); bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize()); - - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); } /** @@ -218,15 +203,13 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { * garbage collections, I avoid the ABA problem by making the kernel of this method * uninterruptible so it can not be interrupted by a safepoint. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeader popUnusedAlignedChunk() { - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); - AlignedHeader result = popUnusedAlignedChunkUninterruptibly(); if (result.isNull()) { return WordFactory.nullPointer(); } else { bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize()); - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); return result; } } @@ -268,7 +251,6 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { /** Acquire an UnalignedHeapChunk from the operating system. */ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - log().string("[HeapChunkProvider.produceUnalignedChunk objectSize: ").unsigned(objectSize).string(" chunkSize: ").zhex(chunkSize).newline(); noteFirstAllocationTime(); UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); @@ -282,8 +264,6 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" returns ").zhex(result).string(" ]").newline(); return result; } @@ -300,10 +280,10 @@ static void consumeUnalignedChunks(UnalignedHeader firstChunk) { freeUnalignedChunkList(firstChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void zap(Header chunk, WordBase value) { Pointer start = HeapChunk.getTopPointer(chunk); Pointer limit = HeapChunk.getEndPointer(chunk); - log().string(" zap chunk: ").zhex(chunk).string(" start: ").zhex(start).string(" limit: ").zhex(limit).string(" value: ").zhex(value).newline(); for (Pointer p = start; p.belowThan(limit); p = p.add(FrameAccess.wordSize())) { p.writeWord(0, value); } @@ -332,6 +312,7 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return continueVisiting; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void noteFirstAllocationTime() { if (firstAllocationTime == 0L) { firstAllocationTime = System.nanoTime(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 9c2be3f9a442..e5817b56068d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -137,6 +137,7 @@ public static UnsignedWord getLargeArrayThreshold() { * Zapping */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 4bf74c62a9f3..d097f56a7284 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -363,6 +363,7 @@ public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } @@ -378,12 +379,14 @@ public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setRememberedSetBit(Object o) { UnsignedWord oldHeader = readHeaderFromObject(o); UnsignedWord newHeader = oldHeader.or(REMEMBERED_SET_BIT); writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } @@ -399,14 +402,18 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -423,11 +430,14 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); @@ -444,12 +454,15 @@ void installForwardingPointer(Object original, Object copy) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { // create forwarding header UnsignedWord forwardHeader; boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); hasShift = true; } else { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 395e39ef03fe..f8d44ad04c42 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -31,8 +31,8 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -73,6 +73,7 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -80,6 +81,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -158,13 +160,13 @@ UnsignedWord getChunkBytes() { return fromBytes.add(toBytes); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") AlignedHeapChunk.AlignedHeader requestAlignedChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - Log.log().string("[! OldGeneration.requestAlignedChunk: failure to allocate aligned chunk!]"); - throw VMError.shouldNotReachHere("Promotion failure"); + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } RememberedSet.get().enableRememberedSetForChunk(chunk); return chunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 31b7ca07cdc2..83c2169b227c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,6 +36,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -79,6 +80,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,6 +89,8 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header @@ -219,6 +223,7 @@ private static boolean processRememberedRef(Reference dr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); UnsignedWord header = ohi.readHeaderFromPointer(referentAddr); @@ -230,6 +235,7 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index b744cae07cea..0eedcf0381b7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,9 +28,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.hub.DynamicHub; import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -39,14 +36,17 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -110,6 +110,7 @@ boolean isEdenSpace() { return age == 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -123,14 +124,17 @@ public boolean isOldSpace() { return age == (HeapParameters.getMaxSurvivorSpaces() + 1); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getAge() { return age; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getNextAgeForPromotion() { return age + 1; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isFromSpace() { return isFromSpace; } @@ -169,6 +173,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemory(UnsignedWord objectSize) { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { return allocateMemoryParallel(objectSize); @@ -187,6 +192,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ @@ -205,14 +211,15 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - @SuppressWarnings("static-method") - private Pointer retractAllocation(UnsignedWord objectSize) { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (newChunk.isNonNull()) { @@ -221,9 +228,10 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk; - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); @@ -329,6 +337,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; extractUnalignedHeapChunkUninterruptibly(uChunk); @@ -397,6 +406,7 @@ private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); @@ -413,6 +423,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); @@ -467,6 +478,7 @@ Object copyAlignedObjectParallel(Object original) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromHeader(Object obj, Word header) { DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header); int encoding = hub.getLayoutEncoding(); @@ -474,6 +486,7 @@ public static UnsignedWord getSizeFromHeader(Object obj, Word header) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); @@ -549,11 +562,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } /** Promote an UnalignedHeapChunk by moving it to this Space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { originalSpace.extractUnalignedHeapChunk(chunk); @@ -577,6 +591,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index fd405abfe1a2..1117387771ca 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -145,6 +145,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index f25fbcf3fcd6..d80e2bf5024c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -29,8 +29,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.heap.ObjectVisitor; @@ -237,6 +237,7 @@ UnsignedWord computeSurvivorObjectBytes() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") public boolean contains(Object object) { if (!HeapImpl.usesImageHeapCardMarking()) { @@ -250,6 +251,7 @@ public boolean contains(Object object) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -265,6 +267,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -304,17 +307,22 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } + // TEMP (chaeubl): this is problematic but it isn't called or? + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + // TEMP (chaeubl): this is problematic but it isn't called or? + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(size); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; if (!alignedChunkFitsInSurvivors()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 757e3e02c324..0f00e4e1d4e5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -26,14 +26,15 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.UnmanagedMemoryUtil; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.os.CommittedMemoryProvider; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; + /** * Synchronized buffer that stores "grey" heap chunks to be scanned. */ @@ -54,21 +55,23 @@ static int wordSize() { this.buffer = malloc(this.size); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { - assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.isOwner(); + assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); if (top >= size) { int oldSize = size; size *= 2; assert top < size; - buffer = realloc(buffer, oldSize, size); + buffer = realloc(buffer, size); } buffer.writeWord(top, ptr); top += wordSize(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer pop() { assert ParallelGC.isInParallelPhase(); - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { if (top > 0) { top -= wordSize(); @@ -87,24 +90,21 @@ boolean isEmpty() { } void release() { - free(buffer, size); + free(buffer); } - @AlwaysInline("GC performance") private static Pointer malloc(int bytes) { - return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(bytes)); + // TEMP (chaeubl): needs proper error handling + return ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bytes)); } - @AlwaysInline("GC performance") - private static Pointer realloc(Pointer orig, int origSize, int newSize) { - Pointer ptr = malloc(newSize); - UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); - free(orig, origSize); - return ptr; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer realloc(Pointer orig, int newSize) { + // TEMP (chaeubl): needs proper error handling + return ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(orig, WordFactory.unsigned(newSize)); } - @AlwaysInline("GC performance") - private static void free(Pointer ptr, int bytes) { - CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(bytes)); + private static void free(Pointer ptr) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index ff25c67e4eff..443c4f63a203 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -26,6 +26,16 @@ package com.oracle.svm.core.genscavenge.parallel; +import java.util.stream.IntStream; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; @@ -33,7 +43,6 @@ import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.jdk.Jvm; @@ -45,15 +54,6 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -import java.util.stream.IntStream; public class ParallelGC { @@ -86,20 +86,24 @@ public static boolean isEnabled() { return SubstrateOptions.UseParallelGC.getValue(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isInParallelPhase() { return singleton().inParallelPhase; } + @Uninterruptible(reason = "Called from a GC worker thread.") public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } + @Uninterruptible(reason = "Called from a GC worker thread.") public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); allocChunkTL.set(chunk); allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); } + @Uninterruptible(reason = "Called from a GC worker thread.") public void push(Pointer ptr) { assert ptr.isNonNull(); if (buffer != null) { @@ -110,6 +114,7 @@ public void push(Pointer ptr) { } } + @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { @@ -142,44 +147,43 @@ private static int getDefaultWorkerCount() { } private Thread startWorkerThread(int n) { - Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.useAsParallelGCThread(); - debugLog().string("WW start ").unsigned(n).newline(); - - while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lock(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); - parPhase.block(); - ++busyWorkers; - debugLog().string("WW run ").unsigned(n).newline(); - } finally { - mutex.unlock(); - } - } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); - } - } - }); + Thread t = new Thread(this::work); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); return t; } + @Uninterruptible(reason = "Called from a GC worker thread.") + private void work() { + VMThreads.SafepointBehavior.useAsParallelGCThread(); + while (true) { + try { + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + parPhase.blockNoTransition(); + ++busyWorkers; + } finally { + mutex.unlock(); + } + } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); + } + } + } + /** * Start parallel phase and wait until all chunks have been processed. * @@ -221,33 +225,31 @@ public boolean waitForIdle() { return true; } - @SuppressWarnings("static-method") - private void scanChunk(Pointer ptr) { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static void scanChunk(Pointer ptr) { if (ptr.isNonNull()) { - debugLog().string("WW scan chunk=").zhex(ptr).newline(); if (ptr.and(UNALIGNED_BIT).notEqual(0)) { - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } else { Pointer start = ptr; AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); if (chunk.equal(ptr)) { start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); } - HeapChunk.walkObjectsFromInline(chunk, start, getVisitor()); + HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } } } - @SuppressWarnings("static-method") - private void scanAllocChunk() { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static void scanAllocChunk() { if (allocChunkNeedsScanning()) { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); - debugLog().string("WW scan alloc=").zhex(allocChunk).string(" from offset ").unsigned(scanOffset).newline(); scannedChunkTL.set(allocChunk); - HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, getVisitor()); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); scannedChunkTL.set(WordFactory.nullPointer()); if (allocChunkTL.get().equal(allocChunk)) { // remember top offset so that we don't scan the same objects again @@ -256,16 +258,12 @@ private void scanAllocChunk() { } } - @SuppressWarnings("static-method") - private boolean allocChunkNeedsScanning() { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static boolean allocChunkNeedsScanning() { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } - private static GreyToBlackObjectVisitor getVisitor() { - return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); - } - @SuppressWarnings("static-method") public void cleanupAfterCollection() { allocChunkTL.set(WordFactory.nullPointer()); @@ -283,6 +281,7 @@ private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } + @Uninterruptible(reason = "Called from a GC worker thread.") static Log debugLog() { return Log.noopLog(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index d244d47c7cb3..b03aa738fdca 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -37,6 +37,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -86,6 +87,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -95,6 +97,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(AlignedHeader chunk) { // Completely clean the card table and the first object table as further objects may be // added later on to this chunk. @@ -118,6 +121,7 @@ public static void clearRememberedSet(AlignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object object, boolean verifyOnly) { Pointer objectPointer = Word.objectToUntrackedPointer(object); AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer); @@ -203,6 +207,7 @@ public static boolean verify(AlignedHeader chunk) { } /** Return the index of an object within the tables of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { UnsignedWord offset = AlignedHeapChunk.getObjectOffset(chunk, objectPointer); return CardTable.memoryOffsetToIndex(offset); @@ -260,18 +265,22 @@ static UnsignedWord getCardTableLimitOffset() { return UnsignedUtils.roundUp(tableLimit, alignment); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(AlignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(Pointer chunk) { return chunk.add(getFirstObjectTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fa707695db94..f5421338675e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; @@ -83,10 +84,12 @@ final class CardTable { private CardTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate @@ -100,6 +103,7 @@ public static void setClean(Pointer table, UnsignedWord index) { table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isDirty(Pointer table, UnsignedWord index) { int entry = readEntry(table, index); return entry == DIRTY_ENTRY; @@ -110,10 +114,12 @@ private static boolean isClean(Pointer table, UnsignedWord index) { return entry == CLEAN_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int readEntry(Pointer table, UnsignedWord index) { return table.readByte(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 40ec1153d282..04281e745f20 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -32,6 +32,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -86,17 +87,20 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); } @@ -107,12 +111,14 @@ public void clearRememberedSet(AlignedHeader chunk) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @@ -131,6 +137,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { if (holderObject == null || object == null) { return; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 2c5f3ff3b141..3f5136bf6e54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; @@ -163,6 +164,7 @@ final class FirstObjectTable { private FirstObjectTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initializeTable(Pointer table, UnsignedWord size) { if (SubstrateUtil.HOSTED) { // Initialize this table unconditionally as this simplifies a few things. @@ -173,12 +175,14 @@ public static void initializeTable(Pointer table, UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean doInitializeTable(Pointer table, UnsignedWord size) { UnmanagedMemoryUtil.fill(table, size, (byte) UNINITIALIZED_ENTRY); return true; } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setTableForObject(Pointer table, UnsignedWord startOffset, UnsignedWord endOffset) { assert startOffset.belowThan(endOffset); UnsignedWord startIndex = memoryOffsetToIndex(startOffset); @@ -328,15 +332,18 @@ private static UnsignedWord getTableSizeForMemoryRange(Pointer memoryStart, Poin * The multiplier from memory offsets to byte offsets into the previous card. This is the * granularity to which I can point to the start of an object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetScale() { return ConfigurationValues.getObjectLayout().getAlignment(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getEntryAtIndex(Pointer table, UnsignedWord index) { return table.readByte(indexToTableOffset(index)); } /** Set the table entry at a given index. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value) { assert isValidEntry(value) : "Invalid entry"; assert isUninitializedIndex(table, index) || getEntryAtIndex(table, index) == value : "Overwriting!"; @@ -363,6 +370,7 @@ private static boolean isMemoryOffsetEntry(int entry) { return MEMORY_OFFSET_MIN <= entry && entry <= MEMORY_OFFSET_MAX; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int biasExponent(int exponent) { assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent + EXPONENT_BIAS; @@ -379,18 +387,22 @@ private static UnsignedWord exponentToOffset(int n) { return WordFactory.unsigned(1L << n); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToTableOffset(UnsignedWord index) { return index.multiply(ENTRY_SIZE_BYTES); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToMemoryOffset(UnsignedWord index) { return index.multiply(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetToEntry(UnsignedWord memoryOffset) { assert memoryOffset.belowThan(BYTES_COVERED_BY_ENTRY) : "Offset out of bounds."; UnsignedWord scaledOffset = memoryOffset.unsignedDivide(memoryOffsetScale()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 8fec4384578c..d0f911951ca4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -34,6 +34,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -85,16 +86,20 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { // Nothing to do. } @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { // Nothing to do. } @@ -105,12 +110,14 @@ public void clearRememberedSet(AlignedHeader chunk) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { // Nothing to do. } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return false; } @@ -127,6 +134,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { // Nothing to do. } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index d6b1725a2967..f2d4e74e00fa 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.Space; @@ -76,28 +77,34 @@ static RememberedSet get() { * Enables remembered set tracking for an aligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(AlignedHeader chunk); /** * Enables remembered set tracking for an unaligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(UnalignedHeader chunk); /** * Enables remembered set tracking for a single object in an aligned chunk. Must be called when * an object is added to the image heap or old generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForObject(AlignedHeader chunk, Object obj); /** Clears the remembered set of an aligned chunk. */ void clearRememberedSet(AlignedHeader chunk); /** Clears the remembered set of an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(UnalignedHeader chunk); /** Checks if remembered set tracking is enabled for an object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean hasRememberedSet(UnsignedWord header); /** @@ -122,6 +129,7 @@ static RememberedSet get() { * tracking is enabled. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void dirtyCardIfNecessary(Object holderObject, Object object); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 1e993fb3f30f..1ee648709ae3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; @@ -59,6 +60,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk) { // The remembered set bit in the header will be set by the code that writes the objects. } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. @@ -67,6 +69,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -75,6 +78,7 @@ public static void clearRememberedSet(UnalignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object obj, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); @@ -132,10 +136,12 @@ static UnsignedWord getObjectIndex() { return WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index b588e0172475..132a8f5d2186 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -367,6 +367,7 @@ private static long remainingNanos(long waitNanos, long startNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { PthreadVMLockSupport.checkResult(Pthread.pthread_cond_signal(getStructPointer()), "pthread_cond_signal"); } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java index c0e97bdb7d1d..5d85820652b1 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java @@ -350,6 +350,7 @@ public long blockNoTransition(long waitNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { Process.NoTransitions.WakeConditionVariable(getStructPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index febd8215da9b..abf3f93fc044 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -75,11 +75,13 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob } /** The minimum alignment of objects (instances and arrays). */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getAlignment() { return objectAlignment; } /** Tests if the given offset or address is aligned according to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAligned(final long value) { return (value % getAlignment() == 0L); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index c203d92c0fe2..af810ff48722 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -29,15 +29,17 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.util.DuplicatedInNativeCode; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.NonmovableByteArrayReader; import com.oracle.svm.core.util.TypedMemoryReader; @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { assert ReferenceMapIndex.denotesValidReferenceMap(referenceMapIndex); assert referenceMapEncoding.isNonNull(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index 5e83ff7a0932..4c3f686e0351 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -63,6 +63,7 @@ protected ObjectHeader() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public abstract DynamicHub dynamicHubFromObjectHeader(Word header); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 86fb9c99d2d3..aa11bdddb98e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; @@ -38,6 +39,8 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } + // TEMP (chaeubl): + @Uninterruptible(reason = "TEMP (chaeubl)", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index bd10eb301a5a..9a827a3b9c3b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -34,16 +34,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.JavaMemoryUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); @@ -55,8 +58,8 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco int gap; do { mapOffset = mapOffset.subtract(2); - gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); - nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); + gap = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); + nrefs = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); for (int i = 0; i < nrefs; i++) { if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java index 68605e09b182..f0c879cd8387 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java @@ -69,6 +69,7 @@ static Reference uncast(Target_java_lang_ref_Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#referent} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset))); } @@ -79,7 +80,7 @@ public static T getReferent(Reference instance) { } /** Write {@link Target_java_lang_ref_Reference#referent}. */ - @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setReferent(Reference instance, Object value) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), value); } @@ -105,6 +106,7 @@ public static void clear(Reference instance) { ObjectAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), null); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentFieldAddress(Reference instance) { return Word.objectToUntrackedPointer(instance).add(WordFactory.unsigned(Target_java_lang_ref_Reference.referentFieldOffset)); } @@ -119,6 +121,7 @@ public static Reference getNextDiscovered(Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#discovered} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getDiscoveredPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset))); } @@ -140,6 +143,7 @@ public static boolean isAnyReferenceFieldOffset(long offset) { } /** Write {@link Target_java_lang_ref_Reference#discovered}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setNextDiscovered(Reference instance, Reference newNext) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset), newNext); } @@ -251,6 +255,7 @@ public static boolean waitForReferenceProcessing() throws InterruptedException { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceClock() { return Target_java_lang_ref_SoftReference.clock; } @@ -262,6 +267,7 @@ public static void updateSoftReferenceClock() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceTimestamp(SoftReference instance) { Target_java_lang_ref_SoftReference ref = SubstrateUtil.cast(instance, Target_java_lang_ref_SoftReference.class); return ref.timestamp; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java index 4989c633cbbe..e80b89687eeb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + public class ReferenceMapIndex { /** * Marker value returned by @@ -44,6 +46,7 @@ public static boolean denotesEmptyReferenceMap(long referenceMapIndex) { return referenceMapIndex == EMPTY_REFERENCE_MAP || referenceMapIndex == NO_REFERENCE_MAP; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean denotesValidReferenceMap(long referenceMapIndex) { return referenceMapIndex != NO_REFERENCE_MAP; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index e4c4e2d9a431..c3fa24fa8ab8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -634,6 +634,7 @@ public DynamicHub getArrayHub() { return arrayHub; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getReferenceMapIndex() { return referenceMapIndex; } @@ -678,6 +679,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getHubType() { return hubType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index fe2a8f6c4509..0aeff0c573fc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; @@ -66,6 +67,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -107,6 +109,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { NonmovableArray referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding(); long referenceMapIndex = objHub.getReferenceMapIndex(); @@ -116,6 +119,7 @@ private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); @@ -124,6 +128,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!Continuation.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); @@ -132,11 +137,13 @@ private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkOther() { throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { int length = ArrayLengthNode.arrayLength(obj); int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 9931f360ea54..61cc8c8b458f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -329,16 +329,19 @@ public static UnsignedWord getMomentarySizeFromObject(Object obj) { return getSizeFromObject(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInGC(Object obj) { return getSizeFromObjectInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); @@ -371,11 +374,13 @@ private static boolean checkOptionalIdentityHashField(Object obj) { return oh.hasOptionalIdentityHashField(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInGC(Object obj) { return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInlineInGC(Object obj) { UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); return Word.objectToUntrackedPointer(obj).add(size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index edb8c09bb17b..838d452afd34 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -435,6 +435,13 @@ public static long abs(long a) { } } + public static class Byte { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int toUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + } + public static class Long { /** Uninterruptible version of {@link java.lang.Long#numberOfLeadingZeros(long)}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java index e20eafb49af8..f048ec19f3a4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java @@ -177,6 +177,7 @@ public long blockNoTransition(long nanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { /* Nothing to do. */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java index 6a5872171303..f85d789026dd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java @@ -98,6 +98,7 @@ public void blockNoTransitionUnspecifiedOwner() { /** * Wakes up a single thread that is waiting on this condition. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { throw VMError.shouldNotReachHere("VMCondition cannot be used during native image generation"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f381b6ed1b4d..9e2be0270d71 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -103,6 +103,7 @@ public int protect(PointerBase start, UnsignedWord nbytes, EnumSet acces } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, false); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 2833afb1377b..326de8007910 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -921,6 +921,7 @@ public static void markThreadAsCrashed() { * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so * it must not keep references to movable heap objects on its stack. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void useAsParallelGCThread() { safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java index 22f004c74e27..8bc6c682604d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java @@ -26,6 +26,9 @@ import org.graalvm.word.Pointer; +import com.oracle.svm.core.Uninterruptible; + +// TEMP (chaeubl): this class could be removed (e.g., combined with native coder) public class TypedMemoryReader { public static int getS1(Pointer ptr) { return ptr.readByte(0); @@ -35,6 +38,7 @@ public static int getS2(Pointer ptr) { return ptr.readShort(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static int getS4(Pointer ptr) { return ptr.readInt(0); } @@ -51,6 +55,7 @@ public static int getU2(Pointer ptr) { return getS2(ptr) & 0xFFFF; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getU4(Pointer ptr) { return getS4(ptr) & 0xFFFFFFFFL; } From 5b16c2910b7cf072370ecf9669a717e0ed978b75 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 15 Mar 2023 17:19:49 +0100 Subject: [PATCH 90/99] Use unattached threads as GC worker threads. --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +- .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../oracle/svm/core/genscavenge/HeapImpl.java | 11 +- .../core/genscavenge/ObjectHeaderImpl.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 6 +- .../svm/core/genscavenge/YoungGeneration.java | 2 - .../genscavenge/parallel/ChunkBuffer.java | 31 +- .../core/genscavenge/parallel/ParallelGC.java | 280 ++++++++++++------ .../parallel/ParallelGCOptions.java | 39 --- .../posix/thread/PosixPlatformThreads.java | 30 ++ .../svm/core/IsolateArgumentParser.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 5 +- .../graal/snippets/CEntryPointSnippets.java | 4 - .../src/com/oracle/svm/core/heap/Heap.java | 2 - .../oracle/svm/core/heap/OutOfMemoryUtil.java | 4 +- .../svm/core/thread/PlatformThreads.java | 33 +++ .../com/oracle/svm/core/thread/Safepoint.java | 6 +- .../com/oracle/svm/core/thread/VMThreads.java | 14 - .../svm/core/util/TypedMemoryReader.java | 1 - 19 files changed, 290 insertions(+), 198 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3dd868593ce1..f6de4b5379f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -146,13 +146,6 @@ public void collect(GCCause cause) { collect(cause, false); } - @SuppressWarnings("static-method") - public void initialize() { - if (ParallelGC.isEnabled()) { - ParallelGC.singleton().startWorkerThreads(); - } - } - public void maybeCollectOnAllocation() { boolean outOfMemory = false; if (hasNeverCollectPolicy()) { @@ -208,6 +201,10 @@ private void collectOperation(CollectionVMOperationData data) { assert VMOperation.isGCInProgress() : "Collection should be a VMOperation."; assert getCollectionEpoch().equal(data.getRequestingEpoch()); + if (SubstrateOptions.UseParallelGC.getValue()) { + ParallelGC.singleton().initialize(); + } + timers.mutator.closeAt(data.getRequestingNanoTime()); startCollectionOrExit(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cad4c249834e..cd13fb0e5989 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -223,7 +223,6 @@ public void noteCopiedReferent() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { - // TEMP (chaeubl): this counter would break unmodifiedReference += 1L; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 76d81c9d14ec..538d223704be 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -42,17 +42,16 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunk; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunkRegistry; import com.oracle.svm.core.SubstrateDiagnostics.ErrorContext; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor; @@ -70,6 +69,7 @@ import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.ReferenceInternals; +import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.locks.VMCondition; @@ -242,11 +242,6 @@ GCImpl getGCImpl() { return gcImpl; } - @Override - public void initGC() { - gcImpl.initialize(); - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index d097f56a7284..5a5ffaee2744 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -411,7 +411,7 @@ Object getForwardedObject(Pointer ptr) { Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { // References compressed with shift have no bits to spare, so the forwarding @@ -435,7 +435,7 @@ void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding @@ -460,7 +460,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe UnsignedWord forwardHeader; boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0eedcf0381b7..9afe74af5c4a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -196,7 +196,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -214,7 +214,7 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } @@ -239,7 +239,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.mutex.unlock(); } if (newChunk.isNonNull()) { - ParallelGC.setAllocationChunk(newChunk); + ParallelGC.singleton().setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index d80e2bf5024c..6a73b2e98912 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -307,14 +307,12 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } - // TEMP (chaeubl): this is problematic but it isn't called or? @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } - // TEMP (chaeubl): this is problematic but it isn't called or? @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 0f00e4e1d4e5..9abcf88dea44 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -28,6 +28,8 @@ import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -50,9 +52,15 @@ static int wordSize() { return ConfigurationValues.getTarget().wordSize; } + @Platforms(Platform.HOSTED_ONLY.class) ChunkBuffer() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { this.size = INITIAL_SIZE; - this.buffer = malloc(this.size); + // TODO (petermz): needs proper error handling + this.buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(this.size)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -62,7 +70,8 @@ void push(Pointer ptr) { int oldSize = size; size *= 2; assert top < size; - buffer = realloc(buffer, size); + // TODO (petermz): needs proper error handling + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); } buffer.writeWord(top, ptr); top += wordSize(); @@ -89,22 +98,8 @@ boolean isEmpty() { return top == 0; } - void release() { - free(buffer); - } - - private static Pointer malloc(int bytes) { - // TEMP (chaeubl): needs proper error handling - return ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bytes)); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static Pointer realloc(Pointer orig, int newSize) { - // TEMP (chaeubl): needs proper error handling - return ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(orig, WordFactory.unsigned(newSize)); - } - - private static void free(Pointer ptr) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + void release() { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 443c4f63a203..59f58f1b1c34 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -26,56 +26,73 @@ package com.oracle.svm.core.genscavenge.parallel; -import java.util.stream.IntStream; +import java.util.function.BooleanSupplier; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.IsolateArgumentParser; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.PlatformThreads; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; +import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.util.VMError; public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; - /** - * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new - * chunk needs to be allocated. - */ - private static final FastThreadLocalWord allocChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); - private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); - private static final FastThreadLocalWord allocChunkScanOffsetTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); + private final CEntryPointLiteral gcWorkerRunFunc; public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); - - private ChunkBuffer buffer; - private Thread[] workers; - private int busyWorkers; + private final ChunkBuffer buffer = new ChunkBuffer(); + + private boolean initialized; + private OSThreadHandlePointer workerThreads; + private GCWorkerThreadState workerStates; + private int numWorkerThreads; + private volatile int busyWorkerThreads; + private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; + @Platforms(Platform.HOSTED_ONLY.class) + public ParallelGC() { + gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); + } + @Fold public static ParallelGC singleton() { return ImageSingletons.lookup(ParallelGC.class); @@ -92,15 +109,17 @@ public static boolean isInParallelPhase() { } @Uninterruptible(reason = "Called from a GC worker thread.") - public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { - return allocChunkTL.get(); + public AlignedHeapChunk.AlignedHeader getAllocationChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + return state.getAllocChunk(); } @Uninterruptible(reason = "Called from a GC worker thread.") - public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { + public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); - allocChunkTL.set(chunk); - allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); + GCWorkerThreadState state = getWorkerThreadState(); + state.setAllocChunk(chunk); + state.setAllocChunkScanOffset(AlignedHeapChunk.getObjectsStartOffset()); } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -117,8 +136,9 @@ public void push(Pointer ptr) { @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); - if (chunk.notEqual(scannedChunkTL.get())) { - UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + GCWorkerThreadState state = getWorkerThreadState(); + if (chunk.notEqual(state.getScannedChunk())) { + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); assert scanOffset.aboveThan(0); if (chunk.getTopOffset().aboveThan(scanOffset)) { push(HeapChunk.asPointer(chunk).add(scanOffset)); @@ -126,61 +146,95 @@ public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { } } - public void startWorkerThreads() { - buffer = new ChunkBuffer(); - int workerCount = getWorkerCount(); - busyWorkers = workerCount; - workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private GCWorkerThreadState getWorkerThreadState() { + if (CurrentIsolate.getCurrentThread().isNull()) { + return PlatformThreads.singleton().getUnmanagedThreadLocalValue(workerStateTL); + } + return workerStates.addressOf(numWorkerThreads); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { + if (initialized) { + return; + } + + initialized = true; + buffer.initialize(); + + workerStateTL = PlatformThreads.singleton().createUnmanagedThreadLocal(); + + numWorkerThreads = getWorkerCount(); + busyWorkerThreads = numWorkerThreads; + + /* Allocate one struct per worker thread and one struct for the main GC thread. */ + int numWorkerStates = numWorkerThreads + 1; + // TODO (petermz): add error handling for allocation + workerStates = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(SizeOf.unsigned(GCWorkerThreadState.class).multiply(numWorkerStates)); + for (int i = 0; i < numWorkerStates; i++) { + workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); + } + + // TODO (petermz): add error handling for allocation + workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); + workerThreads.write(i, thread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getWorkerCount() { - int setting = ParallelGCOptions.ParallelGCThreads.getValue(); + int index = IsolateArgumentParser.getOptionIndex(SubstrateOptions.ParallelGCThreads); + int setting = IsolateArgumentParser.getIntOptionValue(index); int count = setting > 0 ? setting : getDefaultWorkerCount(); - verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(count).string("]").newline(); return count; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getDefaultWorkerCount() { // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() int cpus = Jvm.JVM_ActiveProcessorCount(); return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } - private Thread startWorkerThread(int n) { - Thread t = new Thread(this::work); - t.setName("ParallelGCWorker-" + n); - t.setDaemon(true); - t.start(); - return t; + @Uninterruptible(reason = "Heap base is not set up yet.") + @CEntryPoint(include = UseParallelGC.class, publishAs = CEntryPoint.Publish.NotPublished) + @CEntryPointOptions(prologue = GCWorkerThreadPrologue.class, epilogue = CEntryPointOptions.NoEpilogue.class) + private static void gcWorkerRun(GCWorkerThreadState state) { + try { + ParallelGC.singleton().work(state); + } catch (Throwable e) { + throw VMError.shouldNotReachHere(e); + } } + @NeverInline("Prevent reads from floating up.") @Uninterruptible(reason = "Called from a GC worker thread.") - private void work() { - VMThreads.SafepointBehavior.useAsParallelGCThread(); + private void work(GCWorkerThreadState state) { + PlatformThreads.singleton().setUnmanagedThreadLocalValue(workerStateTL, state); + while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lockNoTransitionUnspecifiedOwner(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - parPhase.blockNoTransition(); - ++busyWorkers; - } finally { - mutex.unlock(); + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (--busyWorkerThreads == 0) { + inParallelPhase = false; + seqPhase.signal(); } + parPhase.blockNoTransition(); + ++busyWorkerThreads; + } finally { + mutex.unlock(); } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); } } @@ -191,16 +245,13 @@ private void work() { * early during application startup. */ public boolean waitForIdle() { - if (workers == null) { - return false; - } - - assert allocChunkTL.get().isNonNull(); - push(HeapChunk.asPointer(allocChunkTL.get())); + GCWorkerThreadState state = getWorkerThreadState(); + assert state.getAllocChunk().isNonNull(); + push(HeapChunk.asPointer(state.getAllocChunk())); mutex.lock(); try { - while (busyWorkers > 0) { // wait for worker threads to become ready + while (busyWorkerThreads > 0) { // wait for worker threads to become ready debugLog().string("PP wait for workers\n"); seqPhase.block(); } @@ -218,9 +269,10 @@ public boolean waitForIdle() { } assert buffer.isEmpty(); - // clean up thread local allocation chunks - for (Thread t : workers) { - allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); + // Reset thread local allocation chunks. + state.setAllocChunk(WordFactory.nullPointer()); + for (int i = 0; i < numWorkerThreads; i++) { + workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); } return true; } @@ -242,49 +294,101 @@ private static void scanChunk(Pointer ptr) { } @Uninterruptible(reason = "Called from a GC worker thread.") - private static void scanAllocChunk() { - if (allocChunkNeedsScanning()) { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + private void scanAllocChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + if (allocChunkNeedsScanning(state)) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); assert scanOffset.aboveThan(0); Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); - scannedChunkTL.set(allocChunk); + state.setScannedChunk(allocChunk); HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); - scannedChunkTL.set(WordFactory.nullPointer()); - if (allocChunkTL.get().equal(allocChunk)) { + state.setScannedChunk(WordFactory.nullPointer()); + if (state.getAllocChunk().equal(allocChunk)) { // remember top offset so that we don't scan the same objects again - allocChunkScanOffsetTL.set(allocChunk.getTopOffset()); + state.setAllocChunkScanOffset(allocChunk.getTopOffset()); } } } @Uninterruptible(reason = "Called from a GC worker thread.") - private static boolean allocChunkNeedsScanning() { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); + private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } @SuppressWarnings("static-method") public void cleanupAfterCollection() { - allocChunkTL.set(WordFactory.nullPointer()); + getWorkerThreadState().setAllocChunk(WordFactory.nullPointer()); } - @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { - buffer.release(); - for (Thread t : workers) { - PlatformThreads.exit(t); - } - } + if (initialized) { + initialized = false; + + buffer.release(); + + // TODO (petermz): signal the worker threads so that they can shut down. + // TODO (petermz): use PlatformThreads.singleton().joinThreadUnmanaged(...) - private static Log verboseGCLog() { - return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); + workerThreads = WordFactory.nullPointer(); + + PlatformThreads.singleton().deleteUnmanagedThreadLocal(workerStateTL); + workerStateTL = WordFactory.nullPointer(); + } } @Uninterruptible(reason = "Called from a GC worker thread.") static Log debugLog() { return Log.noopLog(); } + + @RawStructure + private interface GCWorkerThreadState extends PointerBase { + @RawField + Isolate getIsolate(); + + @RawField + void setIsolate(Isolate value); + + @RawField + AlignedHeapChunk.AlignedHeader getAllocChunk(); + + @RawField + void setAllocChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + AlignedHeapChunk.AlignedHeader getScannedChunk(); + + @RawField + void setScannedChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + UnsignedWord getAllocChunkScanOffset(); + + @RawField + void setAllocChunkScanOffset(UnsignedWord value); + + GCWorkerThreadState addressOf(int index); + } + + private static class GCWorkerThreadPrologue implements CEntryPointOptions.Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(GCWorkerThreadState state) { + CEntryPointSnippets.setHeapBase(state.getIsolate()); + WriteCurrentVMThreadNode.writeCurrentVMThread(WordFactory.nullPointer()); + } + } + + private static class UseParallelGC implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return SubstrateOptions.UseParallelGC.getValue(); + } + } } @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java deleted file mode 100644 index 3857cd35abc2..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.option.RuntimeOptionKey; -import org.graalvm.compiler.options.Option; -import org.graalvm.compiler.options.OptionType; - -import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.Immutable; - -public class ParallelGCOptions { - - @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// - public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); -} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index 3be9f86f296f..b4469e0828c9 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -39,6 +39,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; @@ -267,6 +268,35 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre return status == 0; } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return (ThreadLocalKey) key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unchecked") + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + return (T) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + @Override @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 73d52698a480..9e0bbf3e0905 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -55,7 +55,7 @@ @AutomaticallyRegisteredImageSingleton public class IsolateArgumentParser { private static final RuntimeOptionKey[] OPTIONS = {SubstrateGCOptions.MinHeapSize, SubstrateGCOptions.MaxHeapSize, SubstrateGCOptions.MaxNewSize, - SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData}; + SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData, SubstrateOptions.ParallelGCThreads}; private static final int OPTION_COUNT = OPTIONS.length; private static final CGlobalData OPTION_NAMES = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNames); private static final CGlobalData OPTION_NAME_POSITIONS = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNamePosition); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 24135d51ba16..1c64045f2f5f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.function.Predicate; -import com.oracle.svm.core.util.InterruptImageBuilding; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -66,6 +65,7 @@ import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -326,6 +326,9 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// + public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); + private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { throw new InterruptImageBuilding( diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 9d0ddee83631..1a818b24a79f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -343,10 +343,6 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - // Number of parallel GC workers can be affected by a runtime option, - // so call this after options are parsed - Heap.getHeap().initGC(); - boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now state = success ? FirstIsolateInitStates.SUCCESSFUL : FirstIsolateInitStates.FAILED; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index aca6de0d06e1..42e28322d963 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -81,8 +81,6 @@ protected Heap() { public abstract GC getGC(); - public abstract void initGC(); - public abstract RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport(); /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index aa11bdddb98e..1093b1d57c50 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -39,8 +39,8 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - // TEMP (chaeubl): - @Uninterruptible(reason = "TEMP (chaeubl)", calleeMustBe = false) + // TODO (petermz): may only be executed by the main thread + @Uninterruptible(reason = "TODO (petermz)", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index d22e987f66af..486b3528e35b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -64,10 +64,12 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; +import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; @@ -559,6 +561,26 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.joinThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void closeOSThreadHandle(OSThreadHandle threadHandle) { @@ -1189,8 +1211,19 @@ static void blockedOn(Target_sun_nio_ch_Interruptible b) { } } + @RawStructure public interface OSThreadHandle extends PointerBase { } + + @RawPointerTo(OSThreadHandle.class) + public interface OSThreadHandlePointer extends PointerBase { + void write(int index, OSThreadHandle value); + + OSThreadHandle read(int index); + } + + public interface ThreadLocalKey extends PointerBase { + } } @TargetClass(value = ThreadPoolExecutor.class, innerClass = "Worker") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 729f9da14597..d31707793545 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -760,8 +760,7 @@ private static void waitForSafepoints(String reason) { int safepointBehavior = SafepointBehavior.getSafepointBehaviorVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -906,8 +905,7 @@ public static int countingVMOperation() { int status = StatusSupport.getStatusVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 326de8007910..f22515193945 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -865,9 +865,6 @@ public static class SafepointBehavior { */ static final int THREAD_CRASHED = 2; - /** Similar to THREAD_CRASHED, used by ParallelGC worker threads. */ - static final int PARALLEL_GC_THREAD = 3; - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean ignoresSafepoints() { return safepointBehaviorTL.getVolatile() != ALLOW_SAFEPOINT; @@ -917,15 +914,6 @@ public static void markThreadAsCrashed() { safepointBehaviorTL.setVolatile(THREAD_CRASHED); } - /** - * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so - * it must not keep references to movable heap objects on its stack. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void useAsParallelGCThread() { - safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static String toString(int safepointBehavior) { switch (safepointBehavior) { @@ -935,8 +923,6 @@ public static String toString(int safepointBehavior) { return "PREVENT_VM_FROM_REACHING_SAFEPOINT"; case THREAD_CRASHED: return "THREAD_CRASHED"; - case PARALLEL_GC_THREAD: - return "PARALLEL_GC_THREAD"; default: return "Invalid safepoint behavior"; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java index 8bc6c682604d..43665437715d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java @@ -28,7 +28,6 @@ import com.oracle.svm.core.Uninterruptible; -// TEMP (chaeubl): this class could be removed (e.g., combined with native coder) public class TypedMemoryReader { public static int getS1(Pointer ptr) { return ptr.readByte(0); From f4322206a803dc5e23d4338e87338a8485e06556 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 16 Mar 2023 15:29:53 +0300 Subject: [PATCH 91/99] Cleanup and assertion fixes --- .../svm/core/genscavenge/ChunksAccounting.java | 1 + .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 9 ++------- .../src/com/oracle/svm/core/genscavenge/Space.java | 11 +++++++---- .../svm/core/genscavenge/parallel/ChunkBuffer.java | 2 +- .../svm/core/genscavenge/parallel/ParallelGC.java | 12 +++--------- .../remset/AlignedChunkRememberedSet.java | 1 + .../remset/CardTableBasedRememberedSet.java | 1 + .../core/genscavenge/remset/FirstObjectTable.java | 3 +++ .../svm/core/genscavenge/remset/NoRememberedSet.java | 1 + 9 files changed, 20 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 99b3f184cb1c..02244cea6047 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -97,6 +97,7 @@ void noteAlignedHeapChunk() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteAlignedHeapChunk() { alignedCount--; if (parent != null) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f6de4b5379f2..f0ec2c931709 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1060,9 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) { try { if (isIncremental) { scanGreyObjectsLoop(); - } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { - // GC can happen before parallel worker threads have been started. In this case, - // fall back to serial GC. + } else if (ParallelGC.isEnabled()) { + ParallelGC.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } @@ -1162,10 +1161,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - - if (ParallelGC.isEnabled()) { - ParallelGC.singleton().cleanupAfterCollection(); - } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 9afe74af5c4a..9ee8acae84d4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -115,6 +115,7 @@ public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isSurvivorSpace() { return age > 0 && age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -236,7 +237,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } if (newChunk.isNonNull()) { ParallelGC.singleton().setAllocationChunk(newChunk); @@ -284,6 +285,7 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { assert VMOperation.isGCInProgress() : "Should only be called by the collector."; extractAlignedHeapChunkUninterruptibly(aChunk); @@ -533,11 +535,12 @@ private Object copyAlignedObject(Object originalObj) { } /** Promote an AlignedHeapChunk by moving it to this space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { originalSpace.extractAlignedHeapChunk(chunk); @@ -546,7 +549,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } } @@ -576,7 +579,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 9abcf88dea44..d087fde521d0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -89,7 +89,7 @@ Pointer pop() { return WordFactory.nullPointer(); } } finally { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 59f58f1b1c34..023cc79ad2d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -224,10 +224,10 @@ private void work(GCWorkerThreadState state) { inParallelPhase = false; seqPhase.signal(); } - parPhase.blockNoTransition(); + parPhase.blockNoTransitionUnspecifiedOwner(); ++busyWorkerThreads; } finally { - mutex.unlock(); + mutex.unlockNoTransitionUnspecifiedOwner(); } } @@ -244,7 +244,7 @@ private void work(GCWorkerThreadState state) { * @return false if worker threads have not been started yet. This can happen if GC happens very * early during application startup. */ - public boolean waitForIdle() { + public void waitForIdle() { GCWorkerThreadState state = getWorkerThreadState(); assert state.getAllocChunk().isNonNull(); push(HeapChunk.asPointer(state.getAllocChunk())); @@ -274,7 +274,6 @@ public boolean waitForIdle() { for (int i = 0; i < numWorkerThreads; i++) { workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); } - return true; } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -317,11 +316,6 @@ private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } - @SuppressWarnings("static-method") - public void cleanupAfterCollection() { - getWorkerThreadState().setAllocChunk(WordFactory.nullPointer()); - } - @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { if (initialized) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index b03aa738fdca..f9a39b6dd305 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -113,6 +113,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(AlignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 04281e745f20..6b5e3c3b1100 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -106,6 +106,7 @@ public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 3f5136bf6e54..6a5274856fb3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -350,16 +350,19 @@ private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value table.writeByte(indexToTableOffset(index), (byte) value); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedIndex(Pointer table, UnsignedWord index) { int entry = getEntryAtIndex(table, index); return isUninitializedEntry(entry); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isValidEntry(int entry) { return ENTRY_MIN <= entry && entry <= ENTRY_MAX; } /** May only be used for assertions. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return entry == UNINITIALIZED_ENTRY; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index d0f911951ca4..8cbf01310aaf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -105,6 +105,7 @@ public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { // Nothing to do. } From 22d9dc527ddc5c41c382c4f384487ecc22460c55 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 11:31:06 +0300 Subject: [PATCH 92/99] Addressed minor review comments --- .../core/genscavenge/AlignedHeapChunk.java | 2 +- .../genscavenge/GreyToBlackObjRefVisitor.java | 55 ++++++++++--------- .../core/genscavenge/ObjectHeaderImpl.java | 5 +- .../oracle/svm/core/genscavenge/Space.java | 7 +-- .../genscavenge/parallel/ChunkBuffer.java | 11 ++-- .../core/genscavenge/parallel/ParallelGC.java | 4 +- 6 files changed, 41 insertions(+), 43 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 1a64269b47af..e87e6b922d30 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -118,7 +118,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); - assert newTop.aboveThan(HeapChunk.asPointer(that)); + assert newTop.aboveOrEqual(getObjectsStart(that)); HeapChunk.setTopPointer(that, newTop); return newTop; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cd13fb0e5989..367599924ea9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -155,13 +156,13 @@ public interface Counters extends AutoCloseable { } public static class RealCounters implements Counters { - private long objRef; - private long nullObjRef; - private long nullReferent; - private long forwardedReferent; - private long nonHeapReferent; - private long copiedReferent; - private long unmodifiedReference; + private UninterruptibleUtils.AtomicLong objRef; + private UninterruptibleUtils.AtomicLong nullObjRef; + private UninterruptibleUtils.AtomicLong nullReferent; + private UninterruptibleUtils.AtomicLong forwardedReferent; + private UninterruptibleUtils.AtomicLong nonHeapReferent; + private UninterruptibleUtils.AtomicLong copiedReferent; + private UninterruptibleUtils.AtomicLong unmodifiedReference; RealCounters() { reset(); @@ -169,13 +170,13 @@ public static class RealCounters implements Counters { @Override public void reset() { - objRef = 0L; - nullObjRef = 0L; - nullReferent = 0L; - forwardedReferent = 0L; - nonHeapReferent = 0L; - copiedReferent = 0L; - unmodifiedReference = 0L; + objRef.set(0L); + nullObjRef.set(0L); + nullReferent.set(0L); + forwardedReferent.set(0L); + nonHeapReferent.set(0L); + copiedReferent.set(0L); + unmodifiedReference.set(0L); } @Override @@ -193,50 +194,50 @@ public void close() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { - objRef += 1L; + objRef.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { - nullReferent += 1L; + nullReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { - forwardedReferent += 1L; + forwardedReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { - nonHeapReferent += 1L; + nonHeapReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { - copiedReferent += 1L; + copiedReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { - unmodifiedReference += 1L; + unmodifiedReference.incrementAndGet(); } @Override public void toLog() { Log log = Log.log(); log.string("[GreyToBlackObjRefVisitor.counters:"); - log.string(" objRef: ").signed(objRef); - log.string(" nullObjRef: ").signed(nullObjRef); - log.string(" nullReferent: ").signed(nullReferent); - log.string(" forwardedReferent: ").signed(forwardedReferent); - log.string(" nonHeapReferent: ").signed(nonHeapReferent); - log.string(" copiedReferent: ").signed(copiedReferent); - log.string(" unmodifiedReference: ").signed(unmodifiedReference); + log.string(" objRef: ").signed(objRef.get()); + log.string(" nullObjRef: ").signed(nullObjRef.get()); + log.string(" nullReferent: ").signed(nullReferent.get()); + log.string(" forwardedReferent: ").signed(forwardedReferent.get()); + log.string(" nonHeapReferent: ").signed(nonHeapReferent.get()); + log.string(" copiedReferent: ").signed(copiedReferent.get()); + log.string(" unmodifiedReference: ").signed(unmodifiedReference.get()); log.string("]").newline(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 5a5ffaee2744..f8d4d56ebdac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -66,6 +66,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b00001); private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b00010); private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b00100); + private static final UnsignedWord SHIFT_FORWARD_HEADER = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); /** * Optional: per-object identity hash code state to avoid a fixed field, initially implicitly @@ -440,7 +441,7 @@ void installForwardingPointer(Object original, Object copy) { if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); + forwardHeader = SHIFT_FORWARD_HEADER; ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); @@ -463,7 +464,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); + forwardHeader = SHIFT_FORWARD_HEADER; hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 9ee8acae84d4..5a30cfbfc3d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -447,12 +447,9 @@ Object copyAlignedObjectParallel(Object original) { if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { return null; } - Object copy = copyMemory.toObject(); - if (copy == null) { - return null; - } // Install forwarding pointer into the original header + Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { // We have won the race, now we must copy the object bits. First install the original @@ -462,7 +459,7 @@ Object copyAlignedObjectParallel(Object original) { if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } - int offset = hubOffset + ConfigurationValues.getTarget().wordSize; + int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); if (isOldSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index d087fde521d0..7a99261a7afc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.VMError; /** * Synchronized buffer that stores "grey" heap chunks to be scanned. @@ -44,7 +45,7 @@ public class ChunkBuffer { private static final int INITIAL_SIZE = 1024 * wordSize(); private Pointer buffer; - private int size; + private int size = INITIAL_SIZE; private int top; @Fold @@ -58,20 +59,18 @@ static int wordSize() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void initialize() { - this.size = INITIAL_SIZE; - // TODO (petermz): needs proper error handling - this.buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(this.size)); + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); if (top >= size) { - int oldSize = size; size *= 2; assert top < size; - // TODO (petermz): needs proper error handling buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); } buffer.writeWord(top, ptr); top += wordSize(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 023cc79ad2d9..bcbb1cb850b4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -170,14 +170,14 @@ public void initialize() { /* Allocate one struct per worker thread and one struct for the main GC thread. */ int numWorkerStates = numWorkerThreads + 1; - // TODO (petermz): add error handling for allocation workerStates = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(SizeOf.unsigned(GCWorkerThreadState.class).multiply(numWorkerStates)); + VMError.guarantee(workerStates.isNonNull()); for (int i = 0; i < numWorkerStates; i++) { workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); } - // TODO (petermz): add error handling for allocation workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); + VMError.guarantee(workerThreads.isNonNull()); for (int i = 0; i < numWorkerThreads; i++) { OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); workerThreads.write(i, thread); From 43b944dde642a36260cb062f8b4cf97bbb1be736 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 17:12:52 +0300 Subject: [PATCH 93/99] Adapted to IdentityHash changes --- .../oracle/svm/core/genscavenge/Space.java | 37 +++++++++++-------- .../oracle/svm/core/hub/LayoutEncoding.java | 13 ++++++- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5a30cfbfc3d9..492782153e22 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -441,9 +441,18 @@ Object copyAlignedObjectParallel(Object original) { // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. // If another thread copies the object first, we retract the allocation later. - UnsignedWord size = getSizeFromHeader(original, originalHeader); - assert size.aboveThan(0); - Pointer copyMemory = allocateMemory(size); + UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, false); + UnsignedWord copySize = originalSize; + boolean addIdentityHashField = false; + if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(originalHeader))) { + addIdentityHashField = true; + copySize = LayoutEncoding.getSizeFromHeader(original, originalHeader, true); + } + } + + assert copySize.aboveThan(0); + Pointer copyMemory = allocateMemoryParallel(copySize); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { return null; } @@ -452,15 +461,21 @@ Object copyAlignedObjectParallel(Object original) { Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original - // header + // We have won the race, now we must copy the object bits. First install the original header copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); - UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); + + if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); + offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); + } if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered @@ -471,19 +486,11 @@ Object copyAlignedObjectParallel(Object original) { return copy; } else { // Retract speculatively allocated memory - retractAllocation(size); + retractAllocation(originalSize); return forward; } } - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromHeader(Object obj, Word header) { - DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header); - int encoding = hub.getLayoutEncoding(); - return LayoutEncoding.getSizeFromEncoding(obj, hub, encoding, false); - } - @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 61cc8c8b458f..c9dc2160814d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -348,6 +348,17 @@ public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOp return getSizeFromObjectInline(obj, withOptionalIdHashField); } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getSizeFromHeader(Object obj, Word header, boolean addOptionalIdHashField) { + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + DynamicHub hub = oh.dynamicHubFromObjectHeader(header); + int encoding = hub.getLayoutEncoding(); + boolean withOptionalIdHashField = addOptionalIdHashField || + (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); + return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); + } + @AlwaysInline("Actual inlining decided by callers.") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOptionalIdHashField) { @@ -358,7 +369,7 @@ private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOpti @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromEncoding(Object obj, DynamicHub hub, int encoding, boolean withOptionalIdHashField) { + private static UnsignedWord getSizeFromEncoding(Object obj, DynamicHub hub, int encoding, boolean withOptionalIdHashField) { if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj), withOptionalIdHashField); } else { From c824b1bfd0be6f969bdb40950788d3fbff9e081e Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 17:13:43 +0300 Subject: [PATCH 94/99] Worker thread shutdown routine --- .../svm/core/genscavenge/parallel/ParallelGC.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index bcbb1cb850b4..37b72aa6a2b9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -84,9 +84,10 @@ public class ParallelGC { private OSThreadHandlePointer workerThreads; private GCWorkerThreadState workerStates; private int numWorkerThreads; - private volatile int busyWorkerThreads; + private int busyWorkerThreads; private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; + private volatile boolean shutdown; @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { @@ -225,6 +226,9 @@ private void work(GCWorkerThreadState state) { seqPhase.signal(); } parPhase.blockNoTransitionUnspecifiedOwner(); + if (shutdown) { + return; + } ++busyWorkerThreads; } finally { mutex.unlockNoTransitionUnspecifiedOwner(); @@ -323,8 +327,13 @@ public void tearDown() { buffer.release(); - // TODO (petermz): signal the worker threads so that they can shut down. - // TODO (petermz): use PlatformThreads.singleton().joinThreadUnmanaged(...) + // signal the worker threads so that they can shut down + shutdown = true; + parPhase.broadcast(); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = workerThreads.read(i); + PlatformThreads.singleton().joinThreadUnmanaged(thread, WordFactory.nullPointer()); + } ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); workerThreads = WordFactory.nullPointer(); From 5954d3620c2a5342420b619edcd24d4cf0609522 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Mar 2023 11:46:34 +0300 Subject: [PATCH 95/99] Report OOME on main GC thread only --- .../core/genscavenge/HeapChunkProvider.java | 11 ++++++- .../svm/core/genscavenge/OldGeneration.java | 4 +-- .../oracle/svm/core/genscavenge/Space.java | 8 ++--- .../core/genscavenge/parallel/ParallelGC.java | 33 ++++++++++++++----- .../oracle/svm/core/heap/OutOfMemoryUtil.java | 3 +- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 1aeb21c28ad6..9fcc5490b669 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -95,6 +95,11 @@ public UnsignedWord getBytesInUnusedChunks() { /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { + return produceAlignedChunk(true); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + AlignedHeader produceAlignedChunk(boolean reportOutOfMemory) { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); AlignedHeader result = popUnusedAlignedChunk(); if (result.isNull()) { @@ -102,7 +107,11 @@ AlignedHeader produceAlignedChunk() { noteFirstAllocationTime(); result = (AlignedHeader) CommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); if (result.isNull()) { - throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + if (reportOutOfMemory) { + throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + } else { + throw ALIGNED_OUT_OF_MEMORY_ERROR; + } } AlignedHeapChunk.initialize(result, chunkSize); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index f8d44ad04c42..2bd8eba27116 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -162,9 +162,9 @@ UnsignedWord getChunkBytes() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") - AlignedHeapChunk.AlignedHeader requestAlignedChunk() { + AlignedHeapChunk.AlignedHeader requestAlignedChunk(boolean reportOutOfMemory) { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); + AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(reportOutOfMemory); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 492782153e22..ea73c12e2000 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -222,7 +222,7 @@ private static Pointer retractAllocation(UnsignedWord objectSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(true); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } @@ -235,7 +235,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { ParallelGC.singleton().pushAllocChunk(oldChunk); - newChunk = requestAlignedHeapChunk(); + newChunk = requestAlignedHeapChunk(false); } finally { ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } @@ -599,13 +599,13 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { + private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk(boolean reportOutOfMemory) { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { assert isSurvivorSpace(); chunk = HeapImpl.getHeapImpl().getYoungGeneration().requestAlignedSurvivorChunk(); } else { - chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(); + chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(reportOutOfMemory); } if (chunk.isNonNull()) { appendAlignedHeapChunk(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 37b72aa6a2b9..36e3c41579f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -59,6 +59,7 @@ import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; @@ -88,6 +89,7 @@ public class ParallelGC { private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; private volatile boolean shutdown; + private volatile Throwable error; @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { @@ -218,7 +220,7 @@ private void work(GCWorkerThreadState state) { while (true) { Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { + while (!inParallelPhase || error != null || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { mutex.lockNoTransitionUnspecifiedOwner(); try { if (--busyWorkerThreads == 0) { @@ -235,18 +237,20 @@ private void work(GCWorkerThreadState state) { } } - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); + try { + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable e) { + error = e; + shutdown = true; + } } } /** * Start parallel phase and wait until all chunks have been processed. - * - * @return false if worker threads have not been started yet. This can happen if GC happens very - * early during application startup. */ public void waitForIdle() { GCWorkerThreadState state = getWorkerThreadState(); @@ -272,6 +276,8 @@ public void waitForIdle() { mutex.unlock(); } + haltOnError(); + assert buffer.isEmpty(); // Reset thread local allocation chunks. state.setAllocChunk(WordFactory.nullPointer()); @@ -320,6 +326,17 @@ private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } + /** + * Make sure errors are thrown on main GC thread. + */ + private void haltOnError() { + if (error instanceof OutOfMemoryError oome) { + throw OutOfMemoryUtil.reportOutOfMemoryError(oome); + } else if (error instanceof Error) { + throw VMError.shouldNotReachHere(error); + } + } + @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { if (initialized) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 1093b1d57c50..8ca6e8f525f1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -39,8 +39,7 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - // TODO (petermz): may only be executed by the main thread - @Uninterruptible(reason = "TODO (petermz)", calleeMustBe = false) + @Uninterruptible(reason = "Called from uninterruptible code.", calleeMustBe = false, mayBeInlined = true) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { From cf486e6047174d2af54f0f4329d2bb9259b93ac8 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Mar 2023 13:06:32 +0300 Subject: [PATCH 96/99] Added javadoc for ParallelGC --- .../core/genscavenge/parallel/ParallelGC.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 36e3c41579f3..2b63dd122cd0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -70,6 +70,22 @@ import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.util.VMError; +/** + * A garbage collector that tries to shorten GC pause by executing work on multiple "worker threads". + * Currently the only phase supported is scanning grey objects. Number of worker threads can be set with + * a runtime option. + * + * Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the synchronized + * {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for references to live + * objects to be promoted. For each object being promoted, they speculatively allocate memory for its copy + * in the to-space, then compete to install forwarding pointer in the original object. The winning thread + * proceeds to copy object data, losing threads retract the speculatively allocated memory. + * + * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As allocation + * chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push cycle continues until + * the chunk buffer becomes empty. At this point, worker threads are parked and the GC routine continues on + * the main GC thread. + */ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; From d31e9d775b13fac39a198b341d0bbf3728d95f4a Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Mar 2023 18:53:48 +0300 Subject: [PATCH 97/99] Workaround for @Uninterruptible visitors --- .../core/genscavenge/GreyToBlackObjRefVisitor.java | 10 ++++++++-- .../core/genscavenge/GreyToBlackObjectVisitor.java | 8 +++++++- .../oracle/svm/core/genscavenge/ObjectHeaderImpl.java | 1 + .../src/com/oracle/svm/core/code/CodeInfoEncoder.java | 1 + .../oracle/svm/core/heap/ObjectReferenceVisitor.java | 6 ++++++ .../src/com/oracle/svm/core/heap/ObjectVisitor.java | 11 ++++++++++- 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 367599924ea9..d13e9d9e4edd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -61,13 +61,19 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + return visitObjectReferenceUninterruptibly(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + return visitObjectReferenceUninterruptibly(objRef, innerOffset, compressed, holderObject); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index bc432b866261..329c626fb69d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -56,8 +56,14 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectInline(Object o) { + return visitObjectUninterruptibly(o); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectUninterruptibly(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index f8d4d56ebdac..d9e43accf79e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -210,6 +210,7 @@ public boolean hasOptionalIdentityHashField(Word header) { return header.and(IDHASH_STATE_BITS).equal(inFieldState); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setIdentityHashInField(Object o) { assert VMOperation.isGCInProgress(); VMError.guarantee(!hasFixedIdentityHashField()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index bde7d510f25d..00672fbea363 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -701,6 +701,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { int derivedOffset = NumUtil.safeToInt(objRef.rawValue()); result.markReferenceAtOffset(derivedOffset, derivedOffset - innerOffset, compressed); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java index f51f41d7a382..c424075096b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; @@ -44,12 +45,17 @@ public interface ObjectReferenceVisitor { boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject); /** + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + * * @param innerOffset If the reference is a {@linkplain CodeReferenceMapDecoder derived * reference}, a positive integer that must be subtracted from the address to which * the object reference points in order to get the start of the referenced object. */ @AlwaysInline("GC performance") @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { VMError.guarantee(innerOffset == 0, "visitor does not support derived references"); return visitObjectReference(objRef, compressed, holderObject); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java index ac706669506e..816e276b6d86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + /** * Supply a closure to be applied to Objects. * @@ -38,8 +40,15 @@ public interface ObjectVisitor { @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") boolean visitObject(Object o); - /** Like visitObject(Object), but inlined for performance. */ + /** + * Like visitObject(Object), but inlined for performance. + * + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectInline(Object o) { return visitObject(o); } From 65e7ad2ef3fb8069508327b275cf8a63dec7dde7 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 29 Mar 2023 11:58:57 +0300 Subject: [PATCH 98/99] Fixed crash due to chunk space being null --- .../oracle/svm/core/genscavenge/Space.java | 126 ++++++++---------- 1 file changed, 52 insertions(+), 74 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ea73c12e2000..1fd482e2742b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -46,7 +46,6 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -257,8 +256,13 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { accounting.reset(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Must not interact with garbage collections.") void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { + appendAlignedHeapChunk(aChunk, null); + } + + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. @@ -266,14 +270,26 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); } - appendAlignedHeapChunkUninterruptibly(aChunk); - accounting.noteAlignedHeapChunk(); - } + HeapChunk.setSpace(aChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Should only be called by the collector."; + AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); + AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstAlignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastAlignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteAlignedHeapChunk(); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { AlignedHeapChunk.AlignedHeader oldLast = getLastAlignedHeapChunk(); - HeapChunk.setSpace(aChunk, this); HeapChunk.setPrevious(aChunk, oldLast); HeapChunk.setNext(aChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -283,36 +299,16 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade if (getFirstAlignedHeapChunk().isNull()) { setFirstAlignedHeapChunk(aChunk); } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { - assert VMOperation.isGCInProgress() : "Should only be called by the collector."; - extractAlignedHeapChunkUninterruptibly(aChunk); - accounting.unnoteAlignedHeapChunk(); + accounting.noteAlignedHeapChunk(); } @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { - AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); - AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstAlignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastAlignedHeapChunk(chunkPrev); - } - HeapChunk.setNext(aChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(aChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(aChunk, null); + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + appendUnalignedHeapChunk(uChunk, null); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. @@ -320,14 +316,26 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); } - appendUnalignedHeapChunkUninterruptibly(uChunk); - accounting.noteUnalignedHeapChunk(uChunk); - } + HeapChunk.setSpace(uChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; + UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); + UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstUnalignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastUnalignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteUnalignedHeapChunk(uChunk); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { UnalignedHeapChunk.UnalignedHeader oldLast = getLastUnalignedHeapChunk(); - HeapChunk.setSpace(uChunk, this); HeapChunk.setPrevious(uChunk, oldLast); HeapChunk.setNext(uChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -337,33 +345,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne if (getFirstUnalignedHeapChunk().isNull()) { setFirstUnalignedHeapChunk(uChunk); } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { - assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; - extractUnalignedHeapChunkUninterruptibly(uChunk); - accounting.unnoteUnalignedHeapChunk(uChunk); - } - - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { - UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); - UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstUnalignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastUnalignedHeapChunk(chunkPrev); - } - /* Reset the fields that the result chunk keeps for Space. */ - HeapChunk.setNext(uChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(uChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(uChunk, null); + accounting.noteUnalignedHeapChunk(uChunk); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -547,8 +529,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); @@ -577,8 +558,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); @@ -621,15 +601,13 @@ void absorb(Space src) { AlignedHeapChunk.AlignedHeader aChunk = src.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { AlignedHeapChunk.AlignedHeader next = HeapChunk.getNext(aChunk); - src.extractAlignedHeapChunk(aChunk); - appendAlignedHeapChunk(aChunk); + appendAlignedHeapChunk(aChunk, src); aChunk = next; } UnalignedHeapChunk.UnalignedHeader uChunk = src.getFirstUnalignedHeapChunk(); while (uChunk.isNonNull()) { UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); - src.extractUnalignedHeapChunk(uChunk); - appendUnalignedHeapChunk(uChunk); + appendUnalignedHeapChunk(uChunk, src); uChunk = next; } } From 5b47410fdbb5b45e49e3cdc80b19d03665c6e92f Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 12:19:04 +0200 Subject: [PATCH 99/99] Fix object forwarding. --- .../core/genscavenge/ObjectHeaderImpl.java | 88 ++++++++----------- .../oracle/svm/core/genscavenge/Space.java | 16 ++-- 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index d9e43accf79e..f5e16c2c8702 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -66,7 +66,6 @@ public final class ObjectHeaderImpl extends ObjectHeader { private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b00001); private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b00010); private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b00100); - private static final UnsignedWord SHIFT_FORWARD_HEADER = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); /** * Optional: per-object identity hash code state to avoid a fixed field, initially implicitly @@ -96,7 +95,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { } else { VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); numReservedBits = numMinimumReservedBits + 2; - VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(), "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); } numReservedExtraBits = numReservedBits - numAlignmentBits; @@ -342,8 +341,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition) { - ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { if (partition.isWritable()) { header |= REMEMBERED_SET_BIT.rawValue(); } @@ -413,9 +411,7 @@ Object getForwardedObject(Pointer ptr) { Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { + if (hasShift()) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -435,59 +431,48 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); - UnsignedWord forwardHeader; - if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. - forwardHeader = SHIFT_FORWARD_HEADER; - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); - } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); - } - } else { - forwardHeader = Word.objectToUntrackedPointer(copy); - } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + UnsignedWord forwardHeader = getForwardHeader(copy); + ObjectAccess.writeLong(original, getHubOffset(), forwardHeader.rawValue()); assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); } + /** + * The original header are the 8 bytes at the hub offset (regardless if compressed references + * are used or not). + */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + Object installForwardingPointerParallel(Object original, long eightHeaderBytes, Object copy) { + UnsignedWord forwardHeader = getForwardHeader(copy); + /* Try installing the new header. */ + Pointer originalPtr = Word.objectToUntrackedPointer(original); + long value = originalPtr.compareAndSwapLong(getHubOffset(), eightHeaderBytes, forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); + assert isPointerToForwardedObject(originalPtr); + if (value != eightHeaderBytes) { + return getForwardedObject(originalPtr, WordFactory.unsigned(value)); + } + return copy; + } + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { - // create forwarding header - UnsignedWord forwardHeader; - boolean hasShift = false; + private UnsignedWord getForwardHeader(Object copy) { + UnsignedWord result; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { - forwardHeader = SHIFT_FORWARD_HEADER; - hasShift = true; + UnsignedWord compressedCopy = ReferenceAccess.singleton().getCompressedRepresentation(copy); + if (hasShift()) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + result = compressedCopy.shiftLeft(32).or(WordFactory.unsigned(0x00000000e0e0e0e0L)); } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + result = compressedCopy; } } else { - forwardHeader = Word.objectToUntrackedPointer(copy); + result = Word.objectToUntrackedPointer(copy); } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - forwardHeader = forwardHeader.or(FORWARDED_BIT); - // try installing the new header - Pointer originalPtr = Word.objectToUntrackedPointer(original); - UnsignedWord witness = originalPtr.compareAndSwapWord(getHubOffset(), originalHeader, forwardHeader, LocationIdentity.ANY_LOCATION); - assert isPointerToForwardedObject(originalPtr); - if (witness.notEqual(originalHeader)) { - return getForwardedObject(originalPtr, witness); - } else if (hasShift) { - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); - } - return copy; + assert getHeaderBitsFromHeader(result).equal(0); + return result.or(FORWARDED_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -507,6 +492,11 @@ static int getReferenceSize() { return ConfigurationValues.getObjectLayout().getReferenceSize(); } + @Fold + static boolean hasShift() { + return getReferenceSize() == 4; + } + @Fold static boolean hasFixedIdentityHashField() { return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8ebf281d4d62..c234f9a10de7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -413,9 +413,14 @@ Object copyAlignedObjectParallel(Object original) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); + /* + * Always read 8 bytes at the hub offset so that we can install the forwarding header with + * cmpxchng. + */ Pointer originalMemory = Word.objectToUntrackedPointer(original); int hubOffset = ObjectHeaderImpl.getHubOffset(); - Word originalHeader = originalMemory.readWord(hubOffset); + long eightHeaderBytes = originalMemory.readLong(hubOffset); + Word originalHeader = ObjectHeaderImpl.hasShift() ? WordFactory.unsigned(eightHeaderBytes & 0xFFFFFFFFL) : WordFactory.unsigned(eightHeaderBytes); ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { return ohi.getForwardedObject(originalMemory, originalHeader); @@ -441,15 +446,16 @@ Object copyAlignedObjectParallel(Object original) { // Install forwarding pointer into the original header Object copy = copyMemory.toObject(); - Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); + Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header - copyMemory.writeWord(hubOffset, originalHeader); + // We have won the race, now we must copy the object bits. First install the original + // header + copyMemory.writeLong(hubOffset, eightHeaderBytes); // Copy the rest of original object if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } - int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); + int offset = hubOffset + Long.BYTES; UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) {