From 07234ed89cf83e086a35dd36dfe6ab8b32929c6e Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 2 Apr 2024 12:05:47 +0200 Subject: [PATCH 1/7] Change how stack walks work. Verify that FrameAccess is not used to access native frames. --- .../oracle/svm/core/genscavenge/GCImpl.java | 93 ++-- .../remset/UnalignedChunkRememberedSet.java | 5 +- .../posix/PosixSubstrateSigprofHandler.java | 2 +- .../src/com/oracle/svm/core/FrameAccess.java | 193 ++++++- .../oracle/svm/core/SubstrateDiagnostics.java | 32 +- .../com/oracle/svm/core/SubstrateOptions.java | 3 + .../svm/core/aarch64/AArch64FrameAccess.java | 41 +- .../svm/core/amd64/AMD64FrameAccess.java | 31 -- .../oracle/svm/core/code/CodeInfoAccess.java | 37 +- .../oracle/svm/core/code/CodeInfoDecoder.java | 72 ++- .../svm/core/code/CodeInfoQueryResult.java | 6 +- .../oracle/svm/core/code/CodeInfoTable.java | 6 +- .../svm/core/code/FrameInfoQueryResult.java | 4 + .../svm/core/code/RuntimeCodeInfoMemory.java | 1 + .../core/code/SimpleCodeInfoQueryResult.java | 4 +- .../svm/core/deopt/DeoptimizationRuntime.java | 5 +- .../svm/core/deopt/DeoptimizedFrame.java | 4 +- .../oracle/svm/core/deopt/Deoptimizer.java | 84 +-- .../svm/core/graal/snippets/DeoptTester.java | 5 +- .../svm/core/heap/PodReferenceMapDecoder.java | 7 +- .../core/heap/StoredContinuationAccess.java | 100 ++-- .../svm/core/heap/dump/HeapDumpWriter.java | 4 +- .../jdk/Target_java_lang_StackWalker.java | 257 ++++----- .../svm/core/jdk/VMErrorSubstitutions.java | 6 +- .../oracle/svm/core/jfr/JfrThreadLocal.java | 2 +- .../JfrRecurringCallbackExecutionSampler.java | 2 +- .../svm/core/riscv64/RISCV64FrameAccess.java | 30 - .../svm/core/sampler/SamplerBufferAccess.java | 3 +- .../core/sampler/SamplingStackVisitor.java | 5 +- .../svm/core/snippets/ExceptionUnwind.java | 68 +-- .../com/oracle/svm/core/stack/JavaFrame.java | 60 ++ .../svm/core/stack/JavaFrameAnchors.java | 21 +- .../com/oracle/svm/core/stack/JavaFrames.java | 112 ++++ .../oracle/svm/core/stack/JavaStackWalk.java | 63 ++- .../svm/core/stack/JavaStackWalker.java | 517 ++++++++++++------ .../stack/SubstrateStackIntrospection.java | 9 +- .../svm/core/stack/ThreadStackPrinter.java | 43 +- .../svm/core/thread/PlatformThreads.java | 16 +- .../com/oracle/svm/core/thread/VMThreads.java | 2 +- 39 files changed, 1155 insertions(+), 800 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrame.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2c2fd52eba51..88d8c9d9a166 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -25,9 +25,6 @@ package com.oracle.svm.core.genscavenge; import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; -import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; -import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; - import java.lang.ref.Reference; import org.graalvm.nativeimage.CurrentIsolate; @@ -35,7 +32,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -57,7 +53,6 @@ import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -88,6 +83,9 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.snippets.ImplicitExceptions; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaFrame; +import com.oracle.svm.core.stack.JavaFrames; import com.oracle.svm.core.stack.JavaStackWalk; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackOverflowCheck; @@ -802,9 +800,7 @@ private void blackenStackRoots() { Timer blackenStackRootsTimer = timers.blackenStackRoots.open(); try { Pointer sp = readCallerStackPointer(); - CodePointer ip = readReturnAddress(); - - walkStackRoots(greyToBlackObjRefVisitor, sp, ip, true); + walkStackRoots(greyToBlackObjRefVisitor, sp, true); } finally { blackenStackRootsTimer.close(); } @@ -812,84 +808,63 @@ private void blackenStackRoots() { @AlwaysInline("GC performance") @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", mayBeInlined = true) - static void walkStackRoots(ObjectReferenceVisitor visitor, Pointer currentThreadSp, CodePointer currentThreadIp, boolean visitRuntimeCodeInfo) { - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - JavaStackWalker.initWalk(walk, currentThreadSp, currentThreadIp); - walkStack(walk, visitor, visitRuntimeCodeInfo); + static void walkStackRoots(ObjectReferenceVisitor visitor, Pointer currentThreadSp, boolean visitRuntimeCodeInfo) { + /* + * Walk the current thread (unlike all other threads, it does not have a usable frame + * anchor). + */ + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, CurrentIsolate.getCurrentThread(), sp); + walkStack(CurrentIsolate.getCurrentThread(), walk, visitor, visitRuntimeCodeInfo); /* * Scan the stacks of all the threads. Other threads will be blocked at a safepoint (or in * native code) so they will each have a JavaFrameAnchor in their VMThread. */ - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - if (vmThread == CurrentIsolate.getCurrentThread()) { - /* - * The current thread is already scanned by code above, so we do not have to do - * anything for it here. It might have a JavaFrameAnchor from earlier Java-to-C - * transitions, but certainly not at the top of the stack since it is running this - * code, so just this scan would be incomplete. - */ + for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { + if (thread == CurrentIsolate.getCurrentThread()) { continue; } - if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk, visitor, visitRuntimeCodeInfo); - } + JavaStackWalker.initialize(walk, thread); + walkStack(thread, walk, visitor, visitRuntimeCodeInfo); } } - /** - * This method inlines {@link JavaStackWalker#continueWalk(JavaStackWalk, CodeInfo)} and - * {@link CodeInfoTable#visitObjectReferences}. This avoids looking up the - * {@link SimpleCodeInfoQueryResult} twice per frame, and also ensures that there are no virtual - * calls to a stack frame visitor. - */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", mayBeInlined = true) - private static void walkStack(JavaStackWalk walk, ObjectReferenceVisitor visitor, boolean visitRuntimeCodeInfo) { + private static void walkStack(IsolateThread thread, JavaStackWalk walk, ObjectReferenceVisitor visitor, boolean visitRuntimeCodeInfo) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; - while (true) { - SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - Pointer sp = walk.getSP(); - CodePointer ip = walk.getPossiblyStaleIP(); + while (JavaStackWalker.advance(walk, thread)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "GC must not encounter unknown frames"); /* We are during a GC, so tethering of the CodeInfo is not necessary. */ - CodeInfo codeInfo = CodeInfoAccess.convert(walk.getIPCodeInfo()); - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); + DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(frame); if (deoptFrame == null) { - if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); - } - - CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); - assert Deoptimizer.checkDeoptimized(sp) == null : "We are at a safepoint, so no deoptimization can have happened"; - + Pointer sp = frame.getSP(); + CodeInfo codeInfo = CodeInfoAccess.convert(frame.getIPCodeInfo()); NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); - long referenceMapIndex = queryResult.getReferenceMapIndex(); + long referenceMapIndex = frame.getReferenceMapIndex(); if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) { - throw CodeInfoTable.reportNoReferenceMap(sp, ip, codeInfo); + throw CodeInfoTable.fatalErrorNoReferenceMap(sp, frame.getIP(), codeInfo); } CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); + + if (RuntimeCompilation.isEnabled() && visitRuntimeCodeInfo && !CodeInfoAccess.isAOTImageCode(codeInfo)) { + /* + * Runtime-compiled code that is currently on the stack must be kept alive. So, + * we mark the tether as strongly reachable. The RuntimeCodeCacheWalker will + * handle all other object references later on. + */ + RuntimeCodeInfoAccess.walkTether(codeInfo, visitor); + } } else { /* * This is a deoptimized frame. The DeoptimizedFrame object is stored in the frame, * but it is pinned so we do not need to visit references of the frame. */ } - - if (RuntimeCompilation.isEnabled() && visitRuntimeCodeInfo && !CodeInfoAccess.isAOTImageCode(codeInfo)) { - /* - * Runtime-compiled code that is currently on the stack must be kept alive. So, we - * mark the tether as strongly reachable. The RuntimeCodeCacheWalker will handle all - * other object references later on. - */ - RuntimeCodeInfoAccess.walkTether(codeInfo, visitor); - } - - if (!JavaStackWalker.continueWalk(walk, queryResult, deoptFrame)) { - /* No more caller frame found. */ - return; - } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 4217107b9c26..5fb160529b47 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.genscavenge.remset; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.replacements.nodes.AssertionNode; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -43,6 +41,9 @@ import com.oracle.svm.core.util.HostedByteBufferPointer; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.replacements.nodes.AssertionNode; + final class UnalignedChunkRememberedSet { private UnalignedChunkRememberedSet() { } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java index 237e462cb06f..3a90c84f5369 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java @@ -97,7 +97,7 @@ private static void dispatch(@SuppressWarnings("unused") int signalNumber, @Supp if (tryEnterIsolate()) { CodePointer ip = (CodePointer) RegisterDumper.singleton().getIP(uContext); Pointer sp = (Pointer) RegisterDumper.singleton().getSP(uContext); - tryUninterruptibleStackWalk(ip, sp); + tryUninterruptibleStackWalk(ip, sp, true); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java index 97dd9768976d..f604f9c0e473 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java @@ -24,49 +24,117 @@ */ package com.oracle.svm.core; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaFrameAnchor; +import com.oracle.svm.core.stack.JavaFrameAnchors; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.core.common.type.Stamp; import jdk.graal.compiler.core.common.type.StampFactory; -import jdk.vm.ci.aarch64.AArch64; -import jdk.vm.ci.code.Architecture; +/** + * This class can be used to access physical Java frames. It cannot be used to access virtual Java + * frames, and it must not be used to access physical native frames (we can't assume a specific + * layout for native frames, especially on platform such as aarch64). + *

+ * When accessing a return address, note that the return address belongs to a different frame than + * the stack pointer (SP) because the return address is located at a negative offset relative to SP. + * For Java frames that called native code, the return address is located in the native frame and + * can't be accessed because we can't assume a specific layout for native frames. + */ public abstract class FrameAccess { - @Fold public static FrameAccess singleton() { return ImageSingletons.lookup(FrameAccess.class); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public abstract CodePointer readReturnAddress(Pointer sourceSp); + @Fold + public static int returnAddressSize() { + return singleton().getReturnAddressSize(); + } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public abstract void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public CodePointer readReturnAddress(IsolateThread thread, Pointer sourceSp) { + verifyReturnAddressWithinJavaStack(thread, sourceSp); + return unsafeReadReturnAddress(sourceSp); + } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public abstract Pointer getReturnAddressLocation(Pointer sourceSp); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void writeReturnAddress(IsolateThread thread, Pointer sourceSp, CodePointer newReturnAddress) { + verifyReturnAddressWithinJavaStack(thread, sourceSp); + unsafeReturnAddressLocation(sourceSp).writeWord(0, newReturnAddress); + } - @Fold - public static int returnAddressSize() { - Architecture arch = ConfigurationValues.getTarget().arch; - if (arch instanceof AArch64) { - return wordSize(); - } else { - return arch.getReturnAddressSize(); - } + /** + * This method does not return a pointer because the return address may only be accessed via + * {@link #readReturnAddress} and {@link #writeReturnAddress}. Note that no verification is + * performed, so the returned value may point to a native frame or even outside the thread's + * stack. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public UnsignedWord getReturnAddressLocation(IsolateThread thread, Pointer sourceSp) { + assert thread.isNonNull(); + return unsafeReturnAddressLocation(sourceSp); + } + + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + public CodePointer readReturnAddress(StoredContinuation continuation, Pointer sourceSp) { + verifyReturnAddressWithinStoredContinuation(continuation, sourceSp); + return unsafeReadReturnAddress(sourceSp); + } + + /** + * This method does not return a pointer because the return address may only be accessed via + * {@link #readReturnAddress} and {@link #writeReturnAddress}. Note that no verification is + * performed, so the returned value may point outside the stack of the stored continuation. + */ + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + public UnsignedWord getReturnAddressLocation(StoredContinuation continuation, Pointer sourceSp) { + assert continuation != null; + return unsafeReturnAddressLocation(sourceSp); + } + + /** + * Do not use this method unless absolutely necessary as it does not perform any verification. + * It is very easy to accidentally access a native frame, which can result in hard-to-debug + * transient failures. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public CodePointer unsafeReadReturnAddress(Pointer sourceSp) { + return unsafeReturnAddressLocation(sourceSp).readWord(0); } /** - * Gets the amount by which the stack pointer is adjusted by a call instruction. + * Do not use this method unless absolutely necessary as it does not perform any verification. + * It is very easy to accidentally access a native frame, which can result in hard-to-debug + * transient failures. */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer unsafeReturnAddressLocation(Pointer sourceSp) { + return sourceSp.subtract(returnAddressSize()); + } + @Fold - public abstract int stackPointerAdjustmentOnCall(); + protected int getReturnAddressSize() { + int value = ConfigurationValues.getTarget().arch.getReturnAddressSize(); + assert value > 0; + return value; + } @Fold public static int wordSize() { @@ -81,4 +149,91 @@ public static int uncompressedReferenceSize() { public static Stamp getWordStamp() { return StampFactory.forKind(ConfigurationValues.getTarget().wordJavaKind); } + + /** + * This method is only a best-effort approach as it assumes that the given stack pointer either + * points into a Java frame, to the stack boundary of a native -> Java call, or to the stack + * boundary of a Java -> native call. This code does not detect if the stack pointer points to + * an arbitrary position in a native frame. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void verifyReturnAddressWithinJavaStack(IsolateThread thread, Pointer sourceSp) { + if (SubstrateOptions.VerifyFrameAccess.getValue()) { + verifyReturnAddressWithinJavaStack0(thread, sourceSp, true); + } else { + assert verifyReturnAddressWithinJavaStack0(thread, sourceSp, false); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private boolean verifyReturnAddressWithinJavaStack0(IsolateThread thread, Pointer sourceSp, boolean verifyReturnAddress) { + if (SubstrateDiagnostics.isFatalErrorHandlingThread()) { + return true; + } + + VMError.guarantee(CurrentIsolate.getCurrentThread() == thread || VMOperation.isInProgressAtSafepoint(), "Unsafe access to IsolateThread"); + + UnsignedWord stackBase = VMThreads.StackBase.get(thread); + UnsignedWord stackEnd = VMThreads.StackEnd.get(thread); + + Pointer returnAddressLocation = unsafeReturnAddressLocation(sourceSp); + VMError.guarantee(stackBase.equal(0) || returnAddressLocation.belowThan(stackBase), "Access is outside of the stack memory that is reserved for this thread."); + VMError.guarantee(returnAddressLocation.aboveOrEqual(stackEnd), "Access is outside of the stack memory that is reserved for this thread."); + + Pointer topOfStack = getTopOfStack(thread); + VMError.guarantee(returnAddressLocation.aboveOrEqual(topOfStack), "Access is outside of the part of the stack that is currently used by the thread."); + + if (verifyReturnAddress) { + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); + while (anchor.isNonNull()) { + /* + * If the verification runs for the current thread and if that thread has outdated + * frame anchors, the guarantee below may fail unexpectedly (e.g., we might be in + * the middle of executing safepoint slowpath code for a transition from native to + * Java). However, this is still the best that we can do at the moment (the only + * alternative is a full stack walk, which may run into even more issues). + */ + VMError.guarantee(anchor.getLastJavaSP() != sourceSp, "Potentially accessing a return address that is stored in a native frame."); + anchor = anchor.getPreviousAnchor(); + } + } + + return true; + } + + @NeverInline("Accesses the caller stack pointer") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE) + private static Pointer getTopOfStack(IsolateThread thread) { + if (thread == CurrentIsolate.getCurrentThread()) { + return KnownIntrinsics.readCallerStackPointer(); + } + + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); + VMError.guarantee(anchor.isNonNull(), "When accessing the stack of another thread, the other thread must have a frame anchor."); + return anchor.getLastJavaSP(); + } + + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + private void verifyReturnAddressWithinStoredContinuation(StoredContinuation continuation, Pointer sourceSP) { + if (SubstrateOptions.VerifyFrameAccess.getValue()) { + verifyReturnAddressWithinStoredContinuation0(continuation, sourceSP); + } else { + assert verifyReturnAddressWithinStoredContinuation0(continuation, sourceSP); + } + } + + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + private boolean verifyReturnAddressWithinStoredContinuation0(StoredContinuation continuation, Pointer sourceSP) { + if (SubstrateDiagnostics.isFatalErrorHandlingThread()) { + return true; + } + + UnsignedWord stackEnd = StoredContinuationAccess.getFramesStart(continuation); + UnsignedWord stackBase = StoredContinuationAccess.getFramesEnd(continuation); + + Pointer returnAddressLocation = unsafeReturnAddressLocation(sourceSP); + VMError.guarantee(returnAddressLocation.belowThan(stackBase), "Access is outside of the stack of the stored continuation"); + VMError.guarantee(returnAddressLocation.aboveOrEqual(stackEnd), "Access is outside of the stack of the stored continuation"); + return true; + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index 8acde39240a5..b868e3fc9c00 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -53,10 +53,8 @@ import com.oracle.svm.core.code.FrameInfoQueryResult; import com.oracle.svm.core.code.RuntimeCodeInfoHistory; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; -import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.deopt.DeoptimizationSupport; -import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.RuntimeCompilation; @@ -370,35 +368,11 @@ private static void dumpException(Log log, String currentDumper, Throwable e) { log.resetIndentation().newline(); } - @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") - private static long getTotalFrameSize(Pointer sp, CodePointer ip) { - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); - if (deoptFrame != null) { - return deoptFrame.getSourceTotalFrameSize(); - } - - UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(ip); - if (untetheredInfo.isNonNull()) { - Object tether = CodeInfoAccess.acquireTether(untetheredInfo); - try { - CodeInfo codeInfo = CodeInfoAccess.convert(untetheredInfo, tether); - return getTotalFrameSize0(ip, codeInfo); - } finally { - CodeInfoAccess.releaseTether(untetheredInfo, tether); - } - } - return -1; - } - - @Uninterruptible(reason = "Wrap the now safe call to interruptibly look up the frame size.", calleeMustBe = false) - private static long getTotalFrameSize0(CodePointer ip, CodeInfo codeInfo) { - return CodeInfoAccess.lookupTotalFrameSize(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip)); - } - private static void logFrameAnchors(Log log, IsolateThread thread) { JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); if (anchor.isNull()) { log.string("No anchors").newline(); + return; } int printed = 0; @@ -990,7 +964,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev CodePointer ip = context.getInstructionPointer(); log.string("Stacktrace for the failing thread ").zhex(CurrentIsolate.getCurrentThread()).string(" (A=AOT compiled, J=JIT compiled, D=deoptimized, i=inlined):").indent(true); - boolean success = ThreadStackPrinter.printStacktrace(sp, ip, printVisitors[invocationCount - 1].reset(), log); + boolean success = ThreadStackPrinter.printStacktrace(CurrentIsolate.getCurrentThread(), sp, ip, printVisitors[invocationCount - 1].reset(), log); if (!success && DiagnosticLevel.unsafeOperationsAllowed(maxDiagnosticLevel)) { /* @@ -1021,7 +995,7 @@ private static void startStackWalkInMostLikelyCaller(Log log, int invocationCoun Pointer sp = returnAddressPos.add(FrameAccess.returnAddressSize()); log.newline(); log.string("Starting the stack walk in a possible caller (sp + ").unsigned(sp.subtract(originalSp)).string("):").newline(); - ThreadStackPrinter.printStacktrace(sp, possibleIp, printVisitors[invocationCount - 1].reset(), log); + ThreadStackPrinter.printStacktrace(CurrentIsolate.getCurrentThread(), sp, possibleIp, printVisitors[invocationCount - 1].reset(), log); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 51e733c962dd..7c0bb8fd3231 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -1096,6 +1096,9 @@ protected void onValueUpdate(EconomicMap, Object> values, String ol @Option(help = "Determines if frame anchors are verified at run-time.", type = OptionType.Debug)// public static final HostedOptionKey VerifyFrameAnchors = new HostedOptionKey<>(false); + @Option(help = "Determines if frame accesses are verified at run-time.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyFrameAccess = new HostedOptionKey<>(false); + @SuppressWarnings("unused")// @APIOption(name = "configure-reflection-metadata")// @Option(help = "Enable runtime instantiation of reflection objects for non-invoked methods.", type = OptionType.Expert, deprecated = true)// diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/aarch64/AArch64FrameAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/aarch64/AArch64FrameAccess.java index 431e8687ef1d..f29c1ae5a1e0 100755 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/aarch64/AArch64FrameAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/aarch64/AArch64FrameAccess.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.aarch64; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.function.CodePointer; @@ -35,39 +37,28 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.nodes.aarch64.AArch64XPACNode; -import jdk.graal.compiler.api.replacements.Fold; - @AutomaticallyRegisteredImageSingleton(FrameAccess.class) @Platforms(Platform.AARCH64.class) public class AArch64FrameAccess extends FrameAccess { @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public CodePointer readReturnAddress(Pointer sourceSp) { - /* Read the return address, which is stored immediately below the stack pointer. */ - CodePointer returnAddress = sourceSp.readWord(-returnAddressSize()); - /* Remove the pointer authentication code (PAC), if present. */ - if (SubstrateControlFlowIntegrity.enabled()) { - return AArch64XPACNode.stripAddress(returnAddress); - } - return returnAddress; - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) { - sourceSp.writeWord(-returnAddressSize(), newReturnAddress); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + protected int getReturnAddressSize() { + return wordSize(); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer getReturnAddressLocation(Pointer sourceSp) { - return sourceSp.subtract(returnAddressSize()); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public CodePointer unsafeReadReturnAddress(Pointer sourceSp) { + CodePointer returnAddress = super.unsafeReadReturnAddress(sourceSp); + return stripAddress(returnAddress); } - @Override - @Fold - public int stackPointerAdjustmentOnCall() { - // A call on AArch64 does not touch the SP. - return 0; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static CodePointer stripAddress(CodePointer returnAddress) { + /* Remove the pointer authentication code (PAC), if present. */ + if (SubstrateControlFlowIntegrity.enabled()) { + return AArch64XPACNode.stripAddress(returnAddress); + } + return returnAddress; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/amd64/AMD64FrameAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/amd64/AMD64FrameAccess.java index 20b070dccb0e..22d88b04dbbd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/amd64/AMD64FrameAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/amd64/AMD64FrameAccess.java @@ -26,42 +26,11 @@ import org.graalvm.nativeimage.Platform.AMD64; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.word.Pointer; import com.oracle.svm.core.FrameAccess; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import jdk.graal.compiler.api.replacements.Fold; - @AutomaticallyRegisteredImageSingleton(FrameAccess.class) @Platforms(AMD64.class) public final class AMD64FrameAccess extends FrameAccess { - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public CodePointer readReturnAddress(Pointer sourceSp) { - /* Read the return address, which is stored just below the stack pointer. */ - return sourceSp.readWord(-returnAddressSize()); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) { - sourceSp.writeWord(-returnAddressSize(), newReturnAddress); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer getReturnAddressLocation(Pointer sourceSp) { - return sourceSp.subtract(returnAddressSize()); - } - - @Override - @Fold - public int stackPointerAdjustmentOnCall() { - // A call on AMD64 pushes %rip onto the stack and increments %rsp by wordSize(). - return wordSize(); - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java index 2cb140bb33e0..164cf54dbc18 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java @@ -267,7 +267,14 @@ public static long lookupDeoptimizationEntrypoint(CodeInfo info, long method, lo } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static long lookupTotalFrameSize(CodeInfo info, long ip) { + public static long lookupTotalFrameSize(CodeInfo info, long relativeIP) { + SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); + CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult); + return CodeInfoQueryResult.getTotalFrameSize(codeInfoQueryResult.getEncodedFrameSize()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long lookupTotalFrameSize(CodeInfo info, CodePointer ip) { SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); lookupCodeInfo(info, ip, codeInfoQueryResult); return CodeInfoQueryResult.getTotalFrameSize(codeInfoQueryResult.getEncodedFrameSize()); @@ -279,21 +286,35 @@ public static NonmovableArray getStackReferenceMapEncoding(CodeInfo info) } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static long lookupStackReferenceMapIndex(CodeInfo info, long ip) { - return CodeInfoDecoder.lookupStackReferenceMapIndex(info, ip); + public static long lookupStackReferenceMapIndex(CodeInfo info, long relativeIP) { + return CodeInfoDecoder.lookupStackReferenceMapIndex(info, relativeIP); } - public static void lookupCodeInfo(CodeInfo info, long ip, CodeInfoQueryResult codeInfoQueryResult) { + public static void lookupCodeInfo(CodeInfo info, CodePointer ip, CodeInfoQueryResult codeInfoQueryResult) { lookupCodeInfo(info, ip, codeInfoQueryResult, FrameInfoDecoder.SubstrateConstantAccess); } - public static void lookupCodeInfo(CodeInfo info, long ip, CodeInfoQueryResult codeInfoQueryResult, ConstantAccess constantAccess) { - CodeInfoDecoder.lookupCodeInfo(info, ip, codeInfoQueryResult, constantAccess); + public static void lookupCodeInfo(CodeInfo info, CodePointer ip, CodeInfoQueryResult codeInfoQueryResult, ConstantAccess constantAccess) { + long relativeIP = CodeInfoAccess.relativeIP(info, ip); + CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult, constantAccess); + } + + public static void lookupCodeInfo(CodeInfo info, long relativeIP, CodeInfoQueryResult codeInfoQueryResult, ConstantAccess constantAccess) { + CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult, constantAccess); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void lookupCodeInfo(CodeInfo info, long ip, SimpleCodeInfoQueryResult codeInfoQueryResult) { - CodeInfoDecoder.lookupCodeInfo(info, ip, codeInfoQueryResult); + public static void lookupCodeInfo(CodeInfo info, CodePointer ip, SimpleCodeInfoQueryResult codeInfoQueryResult) { + long relativeIP = CodeInfoAccess.relativeIP(info, ip); + CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long lookupEncodedFrameSize(CodeInfo info, CodePointer ip) { + long relativeIP = CodeInfoAccess.relativeIP(info, ip); + SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); + CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult); + return codeInfoQueryResult.getEncodedFrameSize(); } @Uninterruptible(reason = "Nonmovable object arrays are not visible to GC until installed.") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoDecoder.java index 7ca918d5f46b..e40d8aabf569 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoDecoder.java @@ -85,34 +85,34 @@ private CodeInfoDecoder() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static long lookupCodeInfoEntryOffset(CodeInfo info, long ip) { - long entryIP = lookupEntryIP(ip); - long entryOffset = loadEntryOffset(info, ip); + private static long lookupCodeInfoEntryOffset(CodeInfo info, long relativeIP) { + long entryIP = lookupEntryIP(relativeIP); + long entryOffset = loadEntryOffset(info, relativeIP); do { int entryFlags = loadEntryFlags(info, entryOffset); - if (entryIP == ip) { + if (entryIP == relativeIP) { return entryOffset; } entryIP = advanceIP(info, entryOffset, entryIP); entryOffset = advanceOffset(entryOffset, entryFlags); - } while (entryIP <= ip); + } while (entryIP <= relativeIP); return INVALID_FRAME_INFO_ENTRY_OFFSET; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static long lookupCodeInfoEntryOffsetOrDefault(CodeInfo info, long ip) { + private static long lookupCodeInfoEntryOffsetOrDefault(CodeInfo info, long relativeIP) { int chunksToSearch = 0; while (true) { long defaultFIEntryOffset = INVALID_FRAME_INFO_ENTRY_OFFSET; - long entryIP = UninterruptibleUtils.Math.max(lookupEntryIP(ip) - chunksToSearch * CodeInfoDecoder.indexGranularity(), 0); + long entryIP = UninterruptibleUtils.Math.max(lookupEntryIP(relativeIP) - chunksToSearch * CodeInfoDecoder.indexGranularity(), 0); long entryOffset = loadEntryOffset(info, entryIP); do { int entryFlags = loadEntryFlags(info, entryOffset); int frameInfoFlag = extractFI(entryFlags); defaultFIEntryOffset = frameInfoFlag == FI_DEFAULT_INFO_INDEX_S4 ? entryOffset : defaultFIEntryOffset; - if (entryIP == ip) { + if (entryIP == relativeIP) { if (frameInfoFlag == FI_NO_DEOPT) { /* There is no frame info. Try to find a default one. */ break; @@ -123,7 +123,7 @@ private static long lookupCodeInfoEntryOffsetOrDefault(CodeInfo info, long ip) { entryIP = advanceIP(info, entryOffset, entryIP); entryOffset = advanceOffset(entryOffset, entryFlags); - } while (entryIP <= ip); + } while (entryIP <= relativeIP); if (defaultFIEntryOffset != INVALID_FRAME_INFO_ENTRY_OFFSET) { return defaultFIEntryOffset; @@ -139,14 +139,14 @@ private static long lookupCodeInfoEntryOffsetOrDefault(CodeInfo info, long ip) { } } - static void lookupCodeInfo(CodeInfo info, long ip, CodeInfoQueryResult codeInfoQueryResult, ConstantAccess constantAccess) { - long sizeEncoding = initialSizeEncoding(); - long entryIP = lookupEntryIP(ip); - long entryOffset = loadEntryOffset(info, ip); + static void lookupCodeInfo(CodeInfo info, long relativeIP, CodeInfoQueryResult codeInfoQueryResult, ConstantAccess constantAccess) { + long sizeEncoding = INVALID_SIZE_ENCODING; + long entryIP = lookupEntryIP(relativeIP); + long entryOffset = loadEntryOffset(info, relativeIP); do { int entryFlags = loadEntryFlags(info, entryOffset); sizeEncoding = updateSizeEncoding(info, entryOffset, entryFlags, sizeEncoding); - if (entryIP == ip) { + if (entryIP == relativeIP) { codeInfoQueryResult.encodedFrameSize = sizeEncoding; codeInfoQueryResult.exceptionOffset = loadExceptionOffset(info, entryOffset, entryFlags); codeInfoQueryResult.referenceMapIndex = loadReferenceMapIndex(info, entryOffset, entryFlags); @@ -156,7 +156,7 @@ static void lookupCodeInfo(CodeInfo info, long ip, CodeInfoQueryResult codeInfoQ entryIP = advanceIP(info, entryOffset, entryIP); entryOffset = advanceOffset(entryOffset, entryFlags); - } while (entryIP <= ip); + } while (entryIP <= relativeIP); codeInfoQueryResult.encodedFrameSize = sizeEncoding; codeInfoQueryResult.exceptionOffset = CodeInfoQueryResult.NO_EXCEPTION_OFFSET; @@ -165,14 +165,14 @@ static void lookupCodeInfo(CodeInfo info, long ip, CodeInfoQueryResult codeInfoQ } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static void lookupCodeInfo(CodeInfo info, long ip, SimpleCodeInfoQueryResult codeInfoQueryResult) { - long sizeEncoding = initialSizeEncoding(); - long entryIP = lookupEntryIP(ip); - long entryOffset = loadEntryOffset(info, ip); + static void lookupCodeInfo(CodeInfo info, long relativeIP, SimpleCodeInfoQueryResult codeInfoQueryResult) { + long sizeEncoding = INVALID_SIZE_ENCODING; + long entryIP = lookupEntryIP(relativeIP); + long entryOffset = loadEntryOffset(info, relativeIP); do { int entryFlags = loadEntryFlags(info, entryOffset); sizeEncoding = updateSizeEncoding(info, entryOffset, entryFlags, sizeEncoding); - if (entryIP == ip) { + if (entryIP == relativeIP) { codeInfoQueryResult.setEncodedFrameSize(sizeEncoding); codeInfoQueryResult.setExceptionOffset(loadExceptionOffset(info, entryOffset, entryFlags)); codeInfoQueryResult.setReferenceMapIndex(loadReferenceMapIndex(info, entryOffset, entryFlags)); @@ -181,7 +181,7 @@ static void lookupCodeInfo(CodeInfo info, long ip, SimpleCodeInfoQueryResult cod entryIP = advanceIP(info, entryOffset, entryIP); entryOffset = advanceOffset(entryOffset, entryFlags); - } while (entryIP <= ip); + } while (entryIP <= relativeIP); codeInfoQueryResult.setEncodedFrameSize(sizeEncoding); codeInfoQueryResult.setExceptionOffset(CodeInfoQueryResult.NO_EXCEPTION_OFFSET); @@ -190,8 +190,7 @@ static void lookupCodeInfo(CodeInfo info, long ip, SimpleCodeInfoQueryResult cod static long lookupDeoptimizationEntrypoint(CodeInfo info, long method, long encodedBci, CodeInfoQueryResult codeInfo, ConstantAccess constantAccess) { assert CodeInfoAccess.isAOTImageCode(info); - - long sizeEncoding = initialSizeEncoding(); + long sizeEncoding = INVALID_SIZE_ENCODING; long entryIP = lookupEntryIP(method); long entryOffset = loadEntryOffset(info, method); while (true) { @@ -237,18 +236,18 @@ static long lookupDeoptimizationEntrypoint(CodeInfo info, long method, long enco } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static long lookupStackReferenceMapIndex(CodeInfo info, long ip) { - long entryIP = lookupEntryIP(ip); - long entryOffset = loadEntryOffset(info, ip); + static long lookupStackReferenceMapIndex(CodeInfo info, long relativeIP) { + long entryIP = lookupEntryIP(relativeIP); + long entryOffset = loadEntryOffset(info, relativeIP); do { int entryFlags = loadEntryFlags(info, entryOffset); - if (entryIP == ip) { + if (entryIP == relativeIP) { return loadReferenceMapIndex(info, entryOffset, entryFlags); } entryIP = advanceIP(info, entryOffset, entryIP); entryOffset = advanceOffset(entryOffset, entryFlags); - } while (entryIP <= ip); + } while (entryIP <= relativeIP); return ReferenceMapIndex.NO_REFERENCE_MAP; } @@ -259,14 +258,14 @@ static long indexGranularity() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static long lookupEntryIP(long ip) { - return Long.divideUnsigned(ip, indexGranularity()) * indexGranularity(); + static long lookupEntryIP(long relativeIP) { + return Long.divideUnsigned(relativeIP, indexGranularity()) * indexGranularity(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static long loadEntryOffset(CodeInfo info, long ip) { + private static long loadEntryOffset(CodeInfo info, long relativeIP) { counters().lookupEntryOffsetCount.inc(); - long index = Long.divideUnsigned(ip, indexGranularity()); + long index = Long.divideUnsigned(relativeIP, indexGranularity()); return NonmovableByteArrayReader.getU4(CodeInfoAccess.getCodeInfoIndex(info), index * Integer.BYTES); } @@ -277,14 +276,9 @@ static int loadEntryFlags(CodeInfo info, long curOffset) { return NonmovableByteArrayReader.getU1(CodeInfoAccess.getCodeInfoEncodings(info), curOffset); } - private static final int INVALID_SIZE_ENCODING = 0; + public static final int INVALID_SIZE_ENCODING = 0; private static final int INVALID_FRAME_INFO_ENTRY_OFFSET = -1; - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static int initialSizeEncoding() { - return INVALID_SIZE_ENCODING; - } - @AlwaysInline("Make IP-lookup loop call free") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static long updateSizeEncoding(CodeInfo info, long entryOffset, int entryFlags, long sizeEncoding) { @@ -359,7 +353,7 @@ static long decodeTotalFrameSize(long sizeEncoding) { } private static boolean decodeMethodStart(int entryFlags, long sizeEncoding) { - assert sizeEncoding != initialSizeEncoding() : sizeEncoding; + assert sizeEncoding != INVALID_SIZE_ENCODING : sizeEncoding; switch (extractFS(entryFlags)) { case FS_NO_CHANGE: diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java index e56c4fc63c4f..937867d9dc5b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java @@ -32,7 +32,11 @@ /** * Information about an instruction pointer (IP), created and returned by methods in - * {@link CodeInfoTable}. + * {@link CodeInfoTable}. In situations, where we can't allocate any Java heap memory, we use the + * class {@link SimpleCodeInfoQueryResult} instead. + * + * During a stack walk, this class holds information about a physical Java frame (see + * {@link FrameInfoQueryResult} for the virtual Java frames). */ public class CodeInfoQueryResult { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index b9f145d8c66f..7d23ff9f8f81 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -122,7 +122,7 @@ public static CodeInfoQueryResult lookupCodeInfoQueryResult(CodeInfo info, CodeP } CodeInfoQueryResult result = new CodeInfoQueryResult(); result.ip = absoluteIP; - CodeInfoAccess.lookupCodeInfo(info, CodeInfoAccess.relativeIP(info, absoluteIP), result); + CodeInfoAccess.lookupCodeInfo(info, absoluteIP, result); return result; } @@ -162,13 +162,13 @@ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo referenceMapIndex = CodeInfoAccess.lookupStackReferenceMapIndex(info, CodeInfoAccess.relativeIP(info, ip)); } if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) { - throw reportNoReferenceMap(sp, ip, info); + throw fatalErrorNoReferenceMap(sp, ip, info); } return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); } @Uninterruptible(reason = "Not really uninterruptible, but we are about to fail.", calleeMustBe = false) - public static RuntimeException reportNoReferenceMap(Pointer sp, CodePointer ip, CodeInfo info) { + public static RuntimeException fatalErrorNoReferenceMap(Pointer sp, CodePointer ip, CodeInfo info) { Log.log().string("ip: ").hex(ip).string(" sp: ").hex(sp).string(" info:"); CodeInfoAccess.log(info, Log.log()).newline(); throw VMError.shouldNotReachHere("No reference map information found"); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoQueryResult.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoQueryResult.java index a377061d9831..dee51321bdcc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoQueryResult.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoQueryResult.java @@ -52,6 +52,10 @@ import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ResolvedJavaMethod; +/** + * During a stack walk, this class holds information about a virtual Java frame. It is usually + * referenced by a physical Java frame, see {@link CodeInfoQueryResult}. + */ public class FrameInfoQueryResult { public enum ValueType { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 761481723d48..2315624e1dac 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -335,6 +335,7 @@ public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { for (int i = 0; i < length;) { UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); if (untetheredInfo.isNonNull()) { + /* We are during a GC, so no need for a tether. */ CodeInfo info = CodeInfoAccess.convert(untetheredInfo); callVisitor(visitor, info); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/SimpleCodeInfoQueryResult.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/SimpleCodeInfoQueryResult.java index 5d65e89aa6e7..8c15002c0e86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/SimpleCodeInfoQueryResult.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/SimpleCodeInfoQueryResult.java @@ -28,9 +28,7 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.word.PointerBase; -/** - * Reduced version of {@link CodeInfoQueryResult} as a stack-allocatable (C memory) structure. - */ +/** Reduced version of {@link CodeInfoQueryResult} as a stack-allocatable (C memory) structure. */ @RawStructure public interface SimpleCodeInfoQueryResult extends PointerBase { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizationRuntime.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizationRuntime.java index c4f11f3618ae..cc06f0167db4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizationRuntime.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizationRuntime.java @@ -28,7 +28,7 @@ import java.util.Objects; -import jdk.graal.compiler.graph.NodeSourcePosition; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; @@ -44,6 +44,7 @@ import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; import com.oracle.svm.core.stack.StackOverflowCheck; +import jdk.graal.compiler.graph.NodeSourcePosition; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; @@ -73,7 +74,7 @@ private static void deoptimize(long actionAndReason, SpeculationReason speculati } if (action.doesInvalidateCompilation()) { - Deoptimizer.invalidateMethodOfFrame(sp, speculation); + Deoptimizer.invalidateMethodOfFrame(CurrentIsolate.getCurrentThread(), sp, speculation); } else { Deoptimizer.deoptimizeFrame(sp, false, speculation); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizedFrame.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizedFrame.java index a3f7124eba08..df432e6f6049 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizedFrame.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizedFrame.java @@ -397,7 +397,7 @@ public void takeException() { CodePointer ip = WordFactory.pointer(firstAddressEntry.returnAddress); CodeInfo info = CodeInfoTable.getImageCodeInfo(ip); SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); - CodeInfoAccess.lookupCodeInfo(info, CodeInfoAccess.relativeIP(info, ip), codeInfoQueryResult); + CodeInfoAccess.lookupCodeInfo(info, ip, codeInfoQueryResult); long handler = codeInfoQueryResult.getExceptionOffset(); if (handler == 0) { throwMissingExceptionHandler(info, firstAddressEntry); @@ -409,7 +409,7 @@ public void takeException() { @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Printing out error and then crashing.") private static void throwMissingExceptionHandler(CodeInfo info, ReturnAddress firstAddressEntry) { CodeInfoQueryResult detailedQueryResult = new CodeInfoQueryResult(); - CodeInfoAccess.lookupCodeInfo(info, CodeInfoAccess.relativeIP(info, WordFactory.pointer(firstAddressEntry.returnAddress)), detailedQueryResult); + CodeInfoAccess.lookupCodeInfo(info, WordFactory.pointer(firstAddressEntry.returnAddress), detailedQueryResult); FrameInfoQueryResult frameInfo = detailedQueryResult.getFrameInfo(); throw Deoptimizer.fatalDeoptimizationError("No exception handler registered for deopt target", frameInfo); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java index 10ac226f2334..daa79bf0360e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.deopt; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -75,6 +77,7 @@ import com.oracle.svm.core.monitor.MonitorSupport; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaFrame; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackFrameVisitor; import com.oracle.svm.core.thread.JavaVMOperation; @@ -227,23 +230,37 @@ public static class Options { */ public static boolean testGCinDeoptimizer = false; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static DeoptimizedFrame checkDeoptimized(JavaFrame frame) { + if (DeoptimizationSupport.enabled()) { + return checkDeoptimized0(frame.getSP(), frame.getIP()); + } + return null; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static DeoptimizedFrame checkDeoptimized(IsolateThread thread, Pointer sourceSp) { + if (DeoptimizationSupport.enabled()) { + CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(thread, sourceSp); + return checkDeoptimized0(sourceSp, returnAddress); + } + return null; + } + /** * Checks if a physical stack frame (identified by the stack pointer) was deoptimized, and * returns the {@link DeoptimizedFrame} in that case. */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static DeoptimizedFrame checkDeoptimized(Pointer sourceSp) { - if (DeoptimizationSupport.enabled()) { - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(sourceSp); - /* A frame is deoptimized when the return address was patched to the deoptStub. */ - if (returnAddress.equal(DeoptimizationSupport.getDeoptStubPointer())) { - /* The DeoptimizedFrame instance is stored above the return address. */ - DeoptimizedFrame result = (DeoptimizedFrame) ReferenceAccess.singleton().readObjectAt(sourceSp, true); - if (result == null) { - throw checkDeoptimizedError(sourceSp); - } - return result; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static DeoptimizedFrame checkDeoptimized0(Pointer sourceSp, CodePointer returnAddress) { + /* A frame is deoptimized when the return address was patched to the deoptStub. */ + if (returnAddress.equal(DeoptimizationSupport.getDeoptStubPointer())) { + /* The DeoptimizedFrame instance is stored above the return address. */ + DeoptimizedFrame result = (DeoptimizedFrame) ReferenceAccess.singleton().readObjectAt(sourceSp, true); + if (result == null) { + throw checkDeoptimizedError(sourceSp); } + return result; } return null; } @@ -260,16 +277,16 @@ private static RuntimeException checkDeoptimizedError0(Pointer sourceSp) { } @Uninterruptible(reason = "Prevent stack walks from seeing an inconsistent stack.") - private static void installDeoptimizedFrame(Pointer sourceSp, DeoptimizedFrame deoptimizedFrame) { + private void installDeoptimizedFrame(DeoptimizedFrame deoptimizedFrame) { /* * Replace the return address to the deoptimized method with a pointer to the deoptStub. */ - FrameAccess.singleton().writeReturnAddress(sourceSp, DeoptimizationSupport.getDeoptStubPointer()); + FrameAccess.singleton().writeReturnAddress(targetThread, sourceSp, DeoptimizationSupport.getDeoptStubPointer()); /* - * Store a pointer to the deoptimizedFrame on stack slot above the return address. From this - * point on, the GC will ignore the original source frame content. Instead it just collects - * this pointer to deoptimizedFrame. + * Store a pointer to the deoptimizedFrame in the stack slot above the return address. From + * this point on, the GC will ignore the original source frame content. Instead, it just + * visits the DeoptimizedFrame object. */ ReferenceAccess.singleton().writeObjectAt(sourceSp, deoptimizedFrame, true); } @@ -355,18 +372,17 @@ public boolean visitFrame(Pointer frameSp, CodePointer frameIp, CodeInfo codeInf */ @NeverInline("Inlining of this method would require that we have deopt targets for callees of this method (SVM internals).") public static void deoptimizeFrame(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation) { - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sourceSp); - if (deoptFrame != null) { - /* Already deoptimized, so nothing to do. */ - registerSpeculationFailure(deoptFrame.getSourceInstalledCode(), speculation); - return; - } - /* * Note that the thread needs to be read outside of the VMOperation, since the operation can * run in any different thread. */ IsolateThread targetThread = CurrentIsolate.getCurrentThread(); + DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(targetThread, sourceSp); + if (deoptFrame != null) { + /* Already deoptimized, so nothing to do. */ + registerSpeculationFailure(deoptFrame.getSourceInstalledCode(), speculation); + return; + } DeoptimizeFrameOperation vmOp = new DeoptimizeFrameOperation(sourceSp, ignoreNonDeoptimizable, speculation, targetThread); vmOp.enqueue(); @@ -388,16 +404,12 @@ private static class DeoptimizeFrameOperation extends JavaVMOperation { @Override protected void operate() { - Deoptimizer.deoptimizeFrameOperation(sourceSp, ignoreNonDeoptimizable, speculation, targetThread); + VMOperation.guaranteeInProgress("doDeoptimizeFrame"); + CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(targetThread, sourceSp); + deoptimizeFrame(sourceSp, ignoreNonDeoptimizable, speculation, returnAddress, targetThread); } } - private static void deoptimizeFrameOperation(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation, IsolateThread targetThread) { - VMOperation.guaranteeInProgress("doDeoptimizeFrame"); - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(sourceSp); - deoptimizeFrame(sourceSp, ignoreNonDeoptimizable, speculation, returnAddress, targetThread); - } - @Uninterruptible(reason = "Prevent the GC from freeing the CodeInfo object.") private static void deoptimizeFrame(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodePointer returnAddress, IsolateThread targetThread) { UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(returnAddress); @@ -428,8 +440,8 @@ private static void deoptimize0(Pointer sourceSp, boolean ignoreNonDeoptimizable * Invalidates the {@link InstalledCode} of the method of the given frame. The method must be a * runtime compiled method, since there is not {@link InstalledCode} for native image methods. */ - public static void invalidateMethodOfFrame(Pointer sourceSp, SpeculationReason speculation) { - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(sourceSp); + public static void invalidateMethodOfFrame(IsolateThread thread, Pointer sourceSp, SpeculationReason speculation) { + CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(thread, sourceSp); SubstrateInstalledCode installedCode = CodeInfoTable.lookupInstalledCode(returnAddress); /* * We look up the installedCode before checking if the frame is deoptimized to avoid race @@ -440,7 +452,7 @@ public static void invalidateMethodOfFrame(Pointer sourceSp, SpeculationReason s * installedCode multiple times in case of a race is not a problem because the actual * invalidation is in a VMOperation. */ - DeoptimizedFrame deoptimizedFrame = checkDeoptimized(sourceSp); + DeoptimizedFrame deoptimizedFrame = checkDeoptimized(thread, sourceSp); if (deoptimizedFrame != null) { installedCode = deoptimizedFrame.getSourceInstalledCode(); if (installedCode == null) { @@ -699,7 +711,7 @@ public DeoptimizedFrame getResult() { private DeoptimizedFrame deoptSourceFrameOperation(CodePointer pc, boolean ignoreNonDeoptimizable) { VMOperation.guaranteeInProgress("deoptSourceFrame"); - DeoptimizedFrame existing = checkDeoptimized(sourceSp); + DeoptimizedFrame existing = checkDeoptimized(targetThread, sourceSp); if (existing != null) { /* Already deoptimized, so nothing to do. */ return existing; @@ -770,7 +782,7 @@ private DeoptimizedFrame deoptSourceFrameOperation(CodePointer pc, boolean ignor /* Allocate a buffer to hold the contents of the new target frame. */ DeoptimizedFrame deoptimizedFrame = DeoptimizedFrame.factory(targetContentSize, sourceChunk.getEncodedFrameSize(), CodeInfoTable.lookupInstalledCode(pc), topFrame, relockObjectData, pc); - installDeoptimizedFrame(sourceSp, deoptimizedFrame); + installDeoptimizedFrame(deoptimizedFrame); if (Options.TraceDeoptimization.getValue()) { printDeoptimizedFrame(Log.log(), sourceSp, deoptimizedFrame, frameInfo, false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java index 23b804b431b4..eaa0a4fd3fc0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java @@ -29,8 +29,6 @@ import java.util.HashSet; import java.util.Set; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.options.Option; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; @@ -58,6 +56,9 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalInt; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.options.Option; + /** * Utility class for deoptimization stress test. Used if the DeoptimizeAll option is set. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index 71dbad71f698..ef9854e70b12 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -24,9 +24,6 @@ */ package com.oracle.svm.core.heap; -import jdk.graal.compiler.api.directives.GraalDirectives; -import jdk.graal.compiler.nodes.java.ArrayLengthNode; -import jdk.graal.compiler.word.BarrieredAccess; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -43,6 +40,10 @@ import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.nodes.java.ArrayLengthNode; +import jdk.graal.compiler.word.BarrieredAccess; + public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index 1e2169bb9e02..dd0858fd6dfd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -41,7 +41,6 @@ import com.oracle.svm.core.code.CodeInfoAccess; import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.FrameInfoQueryResult; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.deopt.DeoptimizedFrame; @@ -49,6 +48,8 @@ import com.oracle.svm.core.graal.nodes.NewStoredContinuationNode; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaFrame; +import com.oracle.svm.core.stack.JavaFrames; import com.oracle.svm.core.stack.JavaStackWalk; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackFrameVisitor; @@ -103,6 +104,11 @@ public static Pointer getFramesStart(StoredContinuation s) { return Word.objectToUntrackedPointer(s).add(baseOffset); } + @Uninterruptible(reason = "Prevent GC during accesses via object address.", callerMustBe = true) + public static Pointer getFramesEnd(StoredContinuation s) { + return getFramesStart(s).add(getFramesSizeInBytes(s)); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static CodePointer getIP(StoredContinuation s) { return s.ip; @@ -165,100 +171,66 @@ private static void setIP(StoredContinuation cont, CodePointer ip) { @AlwaysInline("De-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkReferences(Object obj, ObjectReferenceVisitor visitor) { - assert !Heap.getHeap().isInImageHeap(obj) : "StoredContinuations in the image heap are read-only and don't need to be visited"; + public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisitor visitor) { + assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; - StoredContinuation s = (StoredContinuation) obj; - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - if (!initWalk(s, walk)) { - return true; // uninitialized, ignore - } + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, s); + + while (JavaStackWalker.advance(walk, s)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isEntryPoint(frame), "Entry point frames are not supported"); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); + VMError.guarantee(Deoptimizer.checkDeoptimized(frame) == null, "Deoptimized frames are not supported"); - SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - do { - UntetheredCodeInfo untetheredCodeInfo = walk.getIPCodeInfo(); + UntetheredCodeInfo untetheredCodeInfo = frame.getIPCodeInfo(); Object tether = CodeInfoAccess.acquireTether(untetheredCodeInfo); try { CodeInfo codeInfo = CodeInfoAccess.convert(untetheredCodeInfo, tether); - walkFrameReferences(walk, codeInfo, queryResult, visitor, s); + walkFrameReferences(frame, codeInfo, visitor, s); } finally { CodeInfoAccess.releaseTether(untetheredCodeInfo, tether); } - } while (JavaStackWalker.continueWalk(walk, queryResult, null)); + } return true; } @AlwaysInline("De-virtualize calls to visitor.") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkFrames(StoredContinuation s, ContinuationStackFrameVisitor visitor, ContinuationStackFrameVisitorData data) { + public static void walkFrames(StoredContinuation s, ContinuationStackFrameVisitor visitor, ContinuationStackFrameVisitorData data) { assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - if (!initWalk(s, walk)) { - return true; // uninitialized, ignore - } + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, s); + + while (JavaStackWalker.advance(walk, s)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isEntryPoint(frame), "Entry point frames are not supported"); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); + VMError.guarantee(Deoptimizer.checkDeoptimized(frame) == null, "Deoptimized frames are not supported"); - SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - do { - UntetheredCodeInfo untetheredCodeInfo = walk.getIPCodeInfo(); + UntetheredCodeInfo untetheredCodeInfo = frame.getIPCodeInfo(); Object tether = CodeInfoAccess.acquireTether(untetheredCodeInfo); try { - CodeInfo codeInfo = CodeInfoAccess.convert(untetheredCodeInfo); - queryFrameCodeInfo(walk, codeInfo, queryResult); - + CodeInfo codeInfo = CodeInfoAccess.convert(untetheredCodeInfo, tether); NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); - long referenceMapIndex = queryResult.getReferenceMapIndex(); + long referenceMapIndex = frame.getReferenceMapIndex(); if (referenceMapIndex != ReferenceMapIndex.NO_REFERENCE_MAP) { - visitor.visitFrame(data, walk.getSP(), referenceMapEncoding, referenceMapIndex, visitor); + visitor.visitFrame(data, frame.getSP(), referenceMapEncoding, referenceMapIndex, visitor); } } finally { CodeInfoAccess.releaseTether(untetheredCodeInfo, tether); } - } while (JavaStackWalker.continueWalk(walk, queryResult, null)); - - return true; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static void queryFrameCodeInfo(JavaStackWalk walk, CodeInfo codeInfo, SimpleCodeInfoQueryResult queryResult) { - Pointer sp = walk.getSP(); - CodePointer ip = walk.getPossiblyStaleIP(); - - if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, null); } - VMError.guarantee(CodeInfoAccess.isAOTImageCode(codeInfo)); - VMError.guarantee(Deoptimizer.checkDeoptimized(sp) == null); - - CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean initWalk(StoredContinuation s, JavaStackWalk walk) { - CodePointer startIp = getIP(s); - if (startIp.isNull()) { - return false; // uninitialized - } - initWalk(s, walk, startIp); - return true; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void initWalk(StoredContinuation s, JavaStackWalk walk, CodePointer startIp) { - Pointer startSp = getFramesStart(s); - Pointer endSp = getFramesStart(s).add(getSizeInBytes(s)); - JavaStackWalker.initWalkStoredContinuation(walk, startSp, endSp, startIp); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void walkFrameReferences(JavaStackWalk walk, CodeInfo codeInfo, SimpleCodeInfoQueryResult queryResult, ObjectReferenceVisitor visitor, Object holderObject) { - queryFrameCodeInfo(walk, codeInfo, queryResult); - + public static void walkFrameReferences(JavaFrame frame, CodeInfo codeInfo, ObjectReferenceVisitor visitor, Object holderObject) { NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); - long referenceMapIndex = queryResult.getReferenceMapIndex(); + long referenceMapIndex = frame.getReferenceMapIndex(); if (referenceMapIndex != ReferenceMapIndex.NO_REFERENCE_MAP) { - CodeReferenceMapDecoder.walkOffsetsFromPointer(walk.getSP(), referenceMapEncoding, referenceMapIndex, visitor, holderObject); + CodeReferenceMapDecoder.walkOffsetsFromPointer(frame.getSP(), referenceMapEncoding, referenceMapIndex, visitor, holderObject); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java index 30fb293d6c67..6d62eb98e078 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java @@ -1230,12 +1230,12 @@ private void markAsGCRoot(DeoptimizedFrame frame) { private void markStackValuesAsGCRoots(Pointer sp, CodePointer ip, CodeInfo codeInfo) { if (markGCRoots) { SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); + CodeInfoAccess.lookupCodeInfo(codeInfo, ip, queryResult); NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); long referenceMapIndex = queryResult.getReferenceMapIndex(); if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) { - throw CodeInfoTable.reportNoReferenceMap(sp, ip, codeInfo); + throw CodeInfoTable.fatalErrorNoReferenceMap(sp, ip, codeInfo); } CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, this, null); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java index 36b655f6e6d5..e23f2686b8f5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java @@ -34,14 +34,14 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.impl.InternalPlatform; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.annotate.Alias; @@ -50,17 +50,18 @@ import com.oracle.svm.core.annotate.TargetClass; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.code.CodeInfoQueryResult; import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.FrameInfoQueryResult; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.code.UntetheredCodeInfoAccess; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; import com.oracle.svm.core.heap.StoredContinuation; -import com.oracle.svm.core.heap.StoredContinuationAccess; import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaFrame; +import com.oracle.svm.core.stack.JavaFrames; import com.oracle.svm.core.stack.JavaStackFrameVisitor; import com.oracle.svm.core.stack.JavaStackWalk; import com.oracle.svm.core.stack.JavaStackWalker; @@ -139,26 +140,26 @@ private Class getCallerClass() { @Substitute @NeverInline("Starting a stack walk in the caller frame") private T walk(Function, ? extends T> function) { - JavaStackWalk walk = UnsafeStackValue.get(JavaStackWalk.class); + JavaStackWalk walk = UnsafeStackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); AbstractStackFrameSpliterator spliterator; if (ContinuationSupport.isSupported() && continuation != null) { - // walking a yielded continuation + /* Walk a yielded continuation. */ spliterator = new ContinuationSpliterator(walk, contScope, continuation); } else { - // walking a platform thread or mounted continuation + /* Walk the stack of the current thread. */ + IsolateThread isolateThread = CurrentIsolate.getCurrentThread(); Pointer sp = KnownIntrinsics.readCallerStackPointer(); + Pointer endSP = WordFactory.nullPointer(); if (ContinuationSupport.isSupported() && (contScope != null || JavaThreads.isCurrentThreadVirtual())) { var scope = (contScope != null) ? contScope : Target_java_lang_VirtualThread.continuationScope(); var top = Target_jdk_internal_vm_Continuation.getCurrentContinuation(scope); - if (top != null) { // has a delimitation scope - JavaStackWalker.initWalk(walk, sp, ContinuationInternals.getBaseSP(top)); - } else { // scope is not present in current continuation chain or null - JavaStackWalker.initWalk(walk, sp); + if (top != null) { + /* Has a delimitation scope, so we need to stop the stack walk correctly. */ + endSP = ContinuationInternals.getBaseSP(top); } - } else { // walking a platform thread - JavaStackWalker.initWalk(walk, sp); } - spliterator = new StackFrameSpliterator(walk, Thread.currentThread()); + + spliterator = new StackFrameSpliterator(walk, isolateThread, sp, endSP); } try { @@ -185,12 +186,12 @@ public int characteristics() { } @Uninterruptible(reason = "Wraps the now safe call to query frame information.", calleeMustBe = false) - protected FrameInfoQueryResult queryFrameInfo(CodeInfo info, CodePointer ip) { - return CodeInfoTable.lookupCodeInfoQueryResult(info, ip).getFrameInfo(); + protected static CodeInfoQueryResult queryCodeInfoInterruptibly(CodeInfo info, CodePointer ip) { + return CodeInfoTable.lookupCodeInfoQueryResult(info, ip); } - protected DeoptimizedFrame.VirtualFrame curDeoptimizedFrame; - protected FrameInfoQueryResult curRegularFrame; + protected DeoptimizedFrame.VirtualFrame deoptimizedVFrame; + protected FrameInfoQueryResult regularVFrame; @Override public boolean tryAdvance(Consumer action) { @@ -201,152 +202,121 @@ public boolean tryAdvance(Consumer action) { while (true) { /* Check if we have pending virtual frames to process. */ - if (curDeoptimizedFrame != null) { - FrameInfoQueryResult frameInfo = curDeoptimizedFrame.getFrameInfo(); - curDeoptimizedFrame = curDeoptimizedFrame.getCaller(); + if (deoptimizedVFrame != null) { + FrameInfoQueryResult frameInfo = deoptimizedVFrame.getFrameInfo(); + deoptimizedVFrame = deoptimizedVFrame.getCaller(); if (shouldShowFrame(frameInfo, showHiddenFrames, showReflectFrames, showHiddenFrames)) { action.accept(new StackFrameImpl(frameInfo)); return true; } - } else if (curRegularFrame != null) { - FrameInfoQueryResult frameInfo = curRegularFrame; - curRegularFrame = curRegularFrame.getCaller(); + } else if (regularVFrame != null) { + FrameInfoQueryResult frameInfo = regularVFrame; + regularVFrame = frameInfo.getCaller(); if (shouldShowFrame(frameInfo, showHiddenFrames, showReflectFrames, showHiddenFrames)) { action.accept(new StackFrameImpl(frameInfo)); return true; } - } else if (haveMoreFrames()) { - /* No more virtual frames, but we have more physical frames. */ - advancePhysically(); - } else { + } else if (!advancePhysically()) { /* No more physical frames, we are done. */ + invalidate(); return false; } } } - protected boolean shouldShowFrame(FrameInfoQueryResult frameInfo, boolean showLambdaFrames, boolean showReflectFrames, boolean showHiddenFrames) { + private static boolean shouldShowFrame(FrameInfoQueryResult frameInfo, boolean showLambdaFrames, boolean showReflectFrames, boolean showHiddenFrames) { return StackTraceUtils.shouldShowFrame(frameInfo, showLambdaFrames, showReflectFrames, showHiddenFrames); } - protected void invalidate() { - } - - protected void checkState() { - } + protected abstract void invalidate(); - protected abstract boolean haveMoreFrames(); + protected abstract void checkState(); - protected abstract void advancePhysically(); + protected abstract boolean advancePhysically(); } final class ContinuationSpliterator extends AbstractStackFrameSpliterator { private final Target_jdk_internal_vm_ContinuationScope contScope; - private JavaStackWalk walk; + private boolean initialized; + private JavaStackWalk walk; private Target_jdk_internal_vm_Continuation continuation; - private StoredContinuation stored; - - /** - * Because we are interruptible in between walking frames, pointers into the stack become - * invalid if a garbage collection happens and moves the continuation object, so we store - * stack pointers as an offset relative to {@link StoredContinuationAccess#getFramesStart}. - */ - private UnsignedWord spOffset; - private UnsignedWord endSpOffset; ContinuationSpliterator(JavaStackWalk walk, Target_jdk_internal_vm_ContinuationScope contScope, Target_jdk_internal_vm_Continuation continuation) { - walk.setPossiblyStaleIP(WordFactory.nullPointer()); + assert walk.isNonNull(); this.walk = walk; this.contScope = contScope; this.continuation = continuation; } - @Uninterruptible(reason = "Prevent GC while in this method.") - private boolean initWalk() { - assert stored == null; - if (continuation == null || ContinuationInternals.getStoredContinuation(continuation) == null) { - walk.setPossiblyStaleIP(WordFactory.nullPointer()); - return false; - } - if (!StoredContinuationAccess.initWalk(ContinuationInternals.getStoredContinuation(continuation), walk)) { - return false; - } - stored = ContinuationInternals.getStoredContinuation(continuation); - walk.setStartSP(WordFactory.nullPointer()); // not needed, would turn stale - return true; - } - - @Override - protected boolean haveMoreFrames() { - return continuation != null; - } - @Override - protected void advancePhysically() { + protected boolean advancePhysically() { assert continuation != null; - assert curDeoptimizedFrame == null; - curRegularFrame = null; - while (contScope != null && continuation.getScope() != contScope) { - assert stored == null; - continuation = continuation.getParent(); + do { + if (contScope != null) { + /* Navigate to the continuation that matches the scope. */ + while (continuation != null && continuation.getScope() != contScope) { + continuation = continuation.getParent(); + } + } + if (continuation == null) { - return; + return false; } - } - if (!advancePhysically0()) { - continuation = null; - return; - } - if (stored == null) { - continuation = continuation.getParent(); - } + + if (advancePhysically0()) { + return true; + } else { + /* + * We reached the end of the current continuation. Try to continue in the + * parent. + */ + continuation = continuation.getParent(); + initialized = false; + } + } while (continuation != null); + + return false; } @Uninterruptible(reason = "Prevent GC while in this method.") private boolean advancePhysically0() { - if (walk.getPossiblyStaleIP().isNonNull()) { - UnsignedWord framesStart = StoredContinuationAccess.getFramesStart(stored); - walk.setSP((Pointer) framesStart.add(spOffset)); - walk.setEndSP((Pointer) framesStart.add(endSpOffset)); - } else if (!initWalk()) { + StoredContinuation stored = ContinuationInternals.getStoredContinuation(continuation); + if (initialized) { + /* + * Because we are interruptible in between walking frames, pointers into the stored + * continuation become invalid if a garbage collection moves the object. So, we need + * to update all cached stack pointer values before we can continue the walk. + */ + JavaStackWalker.updateStackPointer(walk, stored, 0); + } else { + initialized = true; + JavaStackWalker.initialize(walk, stored); + } + + if (!JavaStackWalker.advance(walk, stored)) { return false; } - VMError.guarantee(Deoptimizer.checkDeoptimized(walk.getSP()) == null); + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isEntryPoint(frame), "Entry point frames are not supported"); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); + VMError.guarantee(Deoptimizer.checkDeoptimized(frame) == null, "Deoptimized frames are not supported"); + + UntetheredCodeInfo untetheredInfo = frame.getIPCodeInfo(); + VMError.guarantee(UntetheredCodeInfoAccess.isAOTImageCode(untetheredInfo)); - CodePointer ip = walk.getPossiblyStaleIP(); - UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(ip); Object tether = CodeInfoAccess.acquireTether(untetheredInfo); - SimpleCodeInfoQueryResult queryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); try { CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); - VMError.guarantee(UntetheredCodeInfoAccess.isAOTImageCode(walk.getIPCodeInfo())); - CodeInfoAccess.lookupCodeInfo(info, CodeInfoAccess.relativeIP(info, ip), queryResult); - - JavaStackWalker.continueWalk(walk, queryResult, null); - if (walk.getSP().belowThan(walk.getEndSP())) { - UnsignedWord framesStart = StoredContinuationAccess.getFramesStart(stored); - spOffset = walk.getSP().subtract(framesStart); - endSpOffset = walk.getEndSP().subtract(framesStart); - } else { - walk.setPossiblyStaleIP(WordFactory.nullPointer()); - spOffset = WordFactory.zero(); - endSpOffset = WordFactory.zero(); - - stored = null; - } - // SPs turn stale when interruptible, null them to be safe - walk.setSP(WordFactory.nullPointer()); - walk.setEndSP(WordFactory.nullPointer()); - - // Interruptible call, so we must finish walking this frame before - curRegularFrame = queryFrameInfo(info, ip); + /* This interruptible call may move the continuation. */ + CodeInfoQueryResult physicalFrame = queryCodeInfoInterruptibly(info, frame.getIP()); + regularVFrame = physicalFrame.getFrameInfo(); } finally { CodeInfoAccess.releaseTether(untetheredInfo, tether); } @@ -355,8 +325,6 @@ private boolean advancePhysically0() { @Override protected void invalidate() { - continuation = null; - stored = null; walk = WordFactory.nullPointer(); } @@ -369,12 +337,19 @@ protected void checkState() { } final class StackFrameSpliterator extends AbstractStackFrameSpliterator { - private final Thread thread; + private final IsolateThread thread; + private final Pointer startSP; + private final Pointer endSP; + + private boolean initialized; private JavaStackWalk walk; - StackFrameSpliterator(JavaStackWalk walk, Thread curThread) { + StackFrameSpliterator(JavaStackWalk walk, IsolateThread thread, Pointer startSP, Pointer endSP) { + this.initialized = false; this.walk = walk; - this.thread = curThread; + this.thread = thread; + this.startSP = startSP; + this.endSP = endSP; } @Override @@ -384,50 +359,44 @@ protected void invalidate() { @Override protected void checkState() { - if (thread != Thread.currentThread()) { - throw new IllegalStateException("Invalid thread"); - } if (walk.isNull()) { throw new IllegalStateException("Stack traversal no longer valid"); + } else if (thread != CurrentIsolate.getCurrentThread()) { + throw new IllegalStateException("Invalid thread"); } } - @Override - protected boolean haveMoreFrames() { - Pointer endSP = walk.getEndSP(); - Pointer curSP = walk.getSP(); - return curSP.isNonNull() && (endSP.isNull() || curSP.belowThan(endSP)); - } - - /** - * Get virtual frames to process in the next loop iteration, then update the physical stack - * walker to the next physical frame to be ready when all virtual frames are processed. - */ @Override @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") - protected void advancePhysically() { - CodePointer ip = FrameAccess.singleton().readReturnAddress(walk.getSP()); - walk.setPossiblyStaleIP(ip); + protected boolean advancePhysically() { + if (!initialized) { + initialized = true; + JavaStackWalker.initialize(walk, thread, startSP, endSP); + } - DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(walk.getSP()); - if (deoptimizedFrame != null) { - curDeoptimizedFrame = deoptimizedFrame.getTopFrame(); - walk.setIPCodeInfo(WordFactory.nullPointer()); - JavaStackWalker.continueWalk(walk, WordFactory.nullPointer()); + if (!JavaStackWalker.advance(walk, thread)) { + return false; + } - } else { - UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(ip); - walk.setIPCodeInfo(untetheredInfo); + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); + DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(frame); + if (deoptimizedFrame != null) { + this.deoptimizedVFrame = deoptimizedFrame.getTopFrame(); + } else { + UntetheredCodeInfo untetheredInfo = frame.getIPCodeInfo(); Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); - curRegularFrame = queryFrameInfo(info, ip); - JavaStackWalker.continueWalk(walk, info); + CodeInfoQueryResult physicalFrame = queryCodeInfoInterruptibly(info, frame.getIP()); + regularVFrame = physicalFrame.getFrameInfo(); } finally { CodeInfoAccess.releaseTether(untetheredInfo, tether); } } + + return true; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java index 89e76ff009a9..e927b8a78b60 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java @@ -26,7 +26,6 @@ import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.NO_ALLOCATION; -import jdk.graal.compiler.nodes.UnreachableNode; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.LogHandler; import org.graalvm.nativeimage.Platforms; @@ -42,10 +41,11 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.stack.StackOverflowCheck; -import com.oracle.svm.core.stack.ThreadStackPrinter; import com.oracle.svm.core.thread.VMThreads.SafepointBehavior; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.UnreachableNode; + @TargetClass(com.oracle.svm.core.util.VMError.class) @Platforms(InternalPlatform.NATIVE_ONLY.class) final class Target_com_oracle_svm_core_util_VMError { @@ -133,8 +133,6 @@ public class VMErrorSubstitutions { @Uninterruptible(reason = "Allow VMError to be used in uninterruptible code.") @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Must not allocate in fatal error handling.") static RuntimeException shouldNotReachHere(CodePointer callerIP, String msg, Throwable ex) { - ThreadStackPrinter.printBacktrace(); - SafepointBehavior.preventSafepoints(); StackOverflowCheck.singleton().disableStackOverflowChecksForFatalError(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java index 2dd6dff650d1..16f270e37af3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java @@ -26,7 +26,6 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; -import com.oracle.svm.core.sampler.SamplerStatistics; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -44,6 +43,7 @@ import com.oracle.svm.core.jfr.events.ThreadStartEvent; import com.oracle.svm.core.sampler.SamplerBuffer; import com.oracle.svm.core.sampler.SamplerSampleWriterData; +import com.oracle.svm.core.sampler.SamplerStatistics; import com.oracle.svm.core.thread.JavaThreads; import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.Target_java_lang_Thread; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java index 184a84e1cdf2..edf7d1ce6f99 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.List; -import com.oracle.svm.core.jfr.JfrExecutionSamplerSupported; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; @@ -45,6 +44,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.jfr.JfrExecutionSamplerSupported; import com.oracle.svm.core.jfr.JfrFeature; import com.oracle.svm.core.jfr.SubstrateJVM; import com.oracle.svm.core.thread.ThreadListenerSupport; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/riscv64/RISCV64FrameAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/riscv64/RISCV64FrameAccess.java index e1546d723d09..f4d41b43400e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/riscv64/RISCV64FrameAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/riscv64/RISCV64FrameAccess.java @@ -26,41 +26,11 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.word.Pointer; import com.oracle.svm.core.FrameAccess; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import jdk.graal.compiler.api.replacements.Fold; - @AutomaticallyRegisteredImageSingleton(FrameAccess.class) @Platforms(Platform.RISCV64.class) public class RISCV64FrameAccess extends FrameAccess { - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public CodePointer readReturnAddress(Pointer sourceSp) { - /* Read the return address, which is stored immediately below the stack pointer */ - return sourceSp.readWord(-returnAddressSize()); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) { - sourceSp.writeWord(-returnAddressSize(), newReturnAddress); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer getReturnAddressLocation(Pointer sourceSp) { - return sourceSp.subtract(returnAddressSize()); - } - - @Override - @Fold - public int stackPointerAdjustmentOnCall() { - // A call on RISCV64 does not touch the SP. - return 0; - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java index 6b9c4b2b83d3..0423aa76e20d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.sampler; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -35,6 +34,8 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.replacements.Fold; + /** * Used to access the raw memory of a {@link SamplerBuffer}. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java index b3e3386196c5..553123082b0e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java @@ -24,12 +24,13 @@ */ package com.oracle.svm.core.sampler; -import com.oracle.svm.core.heap.RestrictHeapAccess; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; class SamplingStackVisitor extends ParameterizedStackFrameVisitor { @@ -49,7 +50,7 @@ protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Deop @Override protected boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data) { - return false; + throw JavaStackWalker.fatalErrorUnknownFrameEncountered(sp, ip); } static class StackTrace { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java index cca5fd95b501..0819e88d2bd5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java @@ -28,6 +28,7 @@ import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.hosted.Feature; @@ -36,19 +37,16 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.code.CodeInfo; -import com.oracle.svm.core.code.CodeInfoAccess; import com.oracle.svm.core.code.CodeInfoQueryResult; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; -import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.deopt.DeoptimizationSupport; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor; +import com.oracle.svm.core.stack.JavaFrame; +import com.oracle.svm.core.stack.JavaFrames; import com.oracle.svm.core.stack.JavaStackWalk; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackOverflowCheck; @@ -57,6 +55,8 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.UnreachableNode; + public abstract class ExceptionUnwind { public static final SubstrateForeignCallDescriptor UNWIND_EXCEPTION_WITHOUT_CALLEE_SAVED_REGISTERS = SnippetRuntime.findForeignCall(ExceptionUnwind.class, @@ -179,65 +179,41 @@ private static void reportUnhandledException(Throwable exception) { @Uninterruptible(reason = "Prevent deoptimization apart from the few places explicitly considered safe for deoptimization") private static void defaultUnwindException(Pointer startSP, boolean fromMethodWithCalleeSavedRegisters) { + IsolateThread thread = CurrentIsolate.getCurrentThread(); boolean hasCalleeSavedRegisters = fromMethodWithCalleeSavedRegisters; - CodePointer startIP = FrameAccess.singleton().readReturnAddress(startSP); /* * callerSP and startIP identify already the caller of the frame that wants to unwind an * exception. So we can start looking for the exception handler immediately in that frame, * without skipping any frames in between. */ - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - JavaStackWalker.initWalk(walk, startSP, startIP); - - while (true) { - SimpleCodeInfoQueryResult codeInfoQueryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - Pointer sp = walk.getSP(); - CodePointer ip = walk.getPossiblyStaleIP(); - - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); - if (deoptFrame == null) { - UntetheredCodeInfo untetheredInfo = walk.getIPCodeInfo(); - if (untetheredInfo.isNull()) { - JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); - return; /* Unreachable code. */ - } - - Object tether = CodeInfoAccess.acquireTether(untetheredInfo); - try { - CodeInfo codeInfo = CodeInfoAccess.convert(untetheredInfo, tether); - - CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), codeInfoQueryResult); - /* - * Frame could have been deoptimized during interruptible lookup above, check - * again. - */ - deoptFrame = Deoptimizer.checkDeoptimized(sp); - } finally { - CodeInfoAccess.releaseTether(untetheredInfo, tether); - } - } + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, thread, startSP); - if (deoptFrame != null && DeoptimizationSupport.enabled()) { + while (JavaStackWalker.advance(walk, thread)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Exception unwinding must not encounter unknown frame"); + + Pointer sp = frame.getSP(); + DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(frame); + if (deoptFrame != null) { /* Deoptimization entry points always have an exception handler. */ deoptTakeExceptionInterruptible(deoptFrame); jumpToHandler(sp, DeoptimizationSupport.getDeoptStubPointer(), hasCalleeSavedRegisters); - return; /* Unreachable code. */ + UnreachableNode.unreachable(); + return; /* Unreachable */ } - long exceptionOffset = codeInfoQueryResult.getExceptionOffset(); + long exceptionOffset = frame.getExceptionOffset(); if (exceptionOffset != CodeInfoQueryResult.NO_EXCEPTION_OFFSET) { - CodePointer handlerIP = (CodePointer) ((UnsignedWord) ip).add(WordFactory.signed(exceptionOffset)); + CodePointer handlerIP = (CodePointer) ((UnsignedWord) frame.getIP()).add(WordFactory.signed(exceptionOffset)); jumpToHandler(sp, handlerIP, hasCalleeSavedRegisters); - return; /* Unreachable code. */ + UnreachableNode.unreachable(); + return; /* Unreachable */ } /* No handler found in this frame, walk to caller frame. */ - hasCalleeSavedRegisters = CodeInfoQueryResult.hasCalleeSavedRegisters(codeInfoQueryResult.getEncodedFrameSize()); - if (!JavaStackWalker.continueWalk(walk, codeInfoQueryResult, deoptFrame)) { - /* No more caller frame found. */ - return; - } + hasCalleeSavedRegisters = CodeInfoQueryResult.hasCalleeSavedRegisters(frame.getEncodedFrameSize()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrame.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrame.java new file mode 100644 index 000000000000..530009eaa7e7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrame.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.stack; + +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.Pointer; + +import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; +import com.oracle.svm.core.code.UntetheredCodeInfo; + +/** + * Represents a physical Java stack frame. + * + * Note that the fields may contain stale values after interruptible code was executed, see + * {@link JavaStackWalk} for more details. + */ +@RawStructure +public interface JavaFrame extends SimpleCodeInfoQueryResult { + @RawField + Pointer getSP(); + + @RawField + void setSP(Pointer sp); + + @RawField + CodePointer getIP(); + + @RawField + void setIP(CodePointer ip); + + @RawField + UntetheredCodeInfo getIPCodeInfo(); + + @RawField + void setIPCodeInfo(UntetheredCodeInfo codeInfo); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java index 11e110dc11de..4d20f6a076e0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.word.WordFactory; @@ -37,6 +38,8 @@ import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; import com.oracle.svm.core.snippets.SnippetRuntime; import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; import com.oracle.svm.core.threadlocal.FastThreadLocal; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; @@ -47,7 +50,10 @@ import jdk.graal.compiler.nodes.extended.ForeignCallNode; /** - * Maintains the linked list of {@link JavaFrameAnchor} for stack walking. + * Maintains the linked list of {@link JavaFrameAnchor} for stack walking. Note that a thread may + * only push/pop/modify a frame anchor, while its thread status is + * {@link StatusSupport#STATUS_IN_JAVA}. This is necessary to guarantee that we can walk the stack + * of other threads consistently while in a VM operation. */ public class JavaFrameAnchors { static final SnippetRuntime.SubstrateForeignCallDescriptor VERIFY_FRAME_ANCHOR_STUB = SnippetRuntime.findForeignCall(JavaFrameAnchors.class, "verifyFrameAnchorStub", NO_SIDE_EFFECT); @@ -83,14 +89,22 @@ public static void popFrameAnchor() { lastAnchorTL.set(prev); } + /** Returns the last Java frame anchor for the current thread, or null if there is none. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static JavaFrameAnchor getFrameAnchor() { return lastAnchorTL.get(); } + /** + * Returns the last Java frame anchor for the given thread, or null if there is none. Note that + * even at a safepoint, there is no guarantee that all stopped {@link IsolateThread}s have a + * Java frame anchor (e.g., threads that are currently attaching don't necesssarily have a frame + * anchor). + */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static JavaFrameAnchor getFrameAnchor(IsolateThread vmThread) { - return lastAnchorTL.get(vmThread); + public static JavaFrameAnchor getFrameAnchor(IsolateThread thread) { + assert thread == CurrentIsolate.getCurrentThread() || VMOperation.isInProgressAtSafepoint(); + return lastAnchorTL.get(thread); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -114,6 +128,7 @@ private static void verifyFrameAnchorStub(boolean newAnchor) { @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static void verifyFrameAnchor(JavaFrameAnchor cur, boolean newAnchor) { + VMError.guarantee(StatusSupport.getStatusVolatile() == StatusSupport.STATUS_IN_JAVA, "Invalid thread status."); VMError.guarantee(cur.getMagicBefore() == JavaFrameAnchor.MAGIC, "Corrupt frame anchor: magic before"); VMError.guarantee(cur.getMagicAfter() == JavaFrameAnchor.MAGIC, "Corrupt frame anchor: magic after"); VMError.guarantee(newAnchor == cur.getLastJavaIP().isNull(), "Corrupt frame anchor: invalid IP"); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java new file mode 100644 index 000000000000..de740da8d5ee --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.stack; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.code.CodeInfoDecoder; +import com.oracle.svm.core.code.CodeInfoQueryResult; +import com.oracle.svm.core.code.CodeInfoTable; +import com.oracle.svm.core.code.UntetheredCodeInfo; +import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.deopt.Deoptimizer; +import com.oracle.svm.core.heap.ReferenceMapIndex; + +public class JavaFrames { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static boolean isUnknownFrame(JavaFrame frame) { + return frame.getIPCodeInfo().isNull() && Deoptimizer.checkDeoptimized(frame) == null; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static boolean isEntryPoint(JavaFrame frame) { + return CodeInfoQueryResult.isEntryPoint(frame.getEncodedFrameSize()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getTotalFrameSize(JavaFrame frame) { + long size = CodeInfoQueryResult.getTotalFrameSize(frame.getEncodedFrameSize()); + assert size > 0; + return WordFactory.unsigned(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static Pointer getCallerSP(JavaFrame frame) { + assert frame.getSP().isNonNull(); + return frame.getSP().add(getTotalFrameSize(frame)); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void clearData(JavaFrame frame) { + frame.setSP(WordFactory.nullPointer()); + frame.setIP(WordFactory.nullPointer()); + frame.setIPCodeInfo(WordFactory.nullPointer()); + + frame.setEncodedFrameSize(CodeInfoDecoder.INVALID_SIZE_ENCODING); + frame.setExceptionOffset(CodeInfoQueryResult.NO_EXCEPTION_OFFSET); + frame.setReferenceMapIndex(ReferenceMapIndex.NO_REFERENCE_MAP); + } + + @Uninterruptible(reason = "Prevent deoptimization and GC.", callerMustBe = true) + public static void setData(JavaFrame frame, Pointer sp, CodePointer ip) { + frame.setSP(sp); + frame.setIP(ip); + + DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(frame); + if (deoptimizedFrame != null) { + frame.setIPCodeInfo(WordFactory.nullPointer()); + frame.setEncodedFrameSize(deoptimizedFrame.getSourceEncodedFrameSize()); + frame.setExceptionOffset(CodeInfoQueryResult.NO_EXCEPTION_OFFSET); + frame.setReferenceMapIndex(ReferenceMapIndex.NO_REFERENCE_MAP); + } else { + UntetheredCodeInfo untetheredCodeInfo = CodeInfoTable.lookupCodeInfo(ip); + frame.setIPCodeInfo(untetheredCodeInfo); + + if (untetheredCodeInfo.isNull()) { + /* Encountered an unknown frame. */ + frame.setEncodedFrameSize(CodeInfoDecoder.INVALID_SIZE_ENCODING); + frame.setExceptionOffset(CodeInfoQueryResult.NO_EXCEPTION_OFFSET); + frame.setReferenceMapIndex(ReferenceMapIndex.NO_REFERENCE_MAP); + } else { + /* Encountered a normal Java frame. */ + Object tether = CodeInfoAccess.acquireTether(untetheredCodeInfo); + try { + CodeInfo info = CodeInfoAccess.convert(untetheredCodeInfo, tether); + CodeInfoAccess.lookupCodeInfo(info, ip, frame); + } finally { + CodeInfoAccess.releaseTether(untetheredCodeInfo, tether); + } + } + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java index 2261873f7476..a9dcfebdc4ac 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java @@ -30,41 +30,44 @@ import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.code.UntetheredCodeInfo; +import com.oracle.svm.core.heap.StoredContinuation; /** - * An in-progress Java stack walk. + * An in-progress stack walk over physical Java frames. The size of this data structure can be + * queried via {@link JavaStackWalker#sizeOfJavaStackWalk}. Only some fields of this data structure + * may be accessed directly (see helper methods in {@link JavaStackWalker}). */ @RawStructure public interface JavaStackWalk extends PointerBase { - @RawField - Pointer getSP(); - - @RawField - void setSP(Pointer sp); - - /** - * The IP can be stale (outdated) if since its retrieval, {@linkplain Uninterruptible - * interruptible} code has executed, during which a deoptimization can have happened. - */ - @RawField - CodePointer getPossiblyStaleIP(); - - @RawField - void setPossiblyStaleIP(CodePointer ip); +} +/** + * The actual implementation. Most stack-walk related fields may only be accessed in + * {@link JavaStackWalker}. + * + * Note that this data structure stores some information about the current physical stack frame + * (e.g., SP, IP, frame size) and also some state that is only needed for the stack walk. + * + * If interruptible code is executed while a stack walk is in progress, IP and code-related fields + * in this data structure may contain stale/outdated values (code may get deoptimized). + * + * If interruptible code is executed while doing a stack walk for a {@link StoredContinuation}, all + * SP-related fields may contain stale/outdated values (the GC may move the + * {@link StoredContinuation}). + */ +@RawStructure +interface JavaStackWalkImpl extends JavaStackWalk { @RawField - UntetheredCodeInfo getIPCodeInfo(); + boolean getStarted(); @RawField - void setIPCodeInfo(UntetheredCodeInfo codeInfo); + void setStarted(boolean value); @RawField - JavaFrameAnchor getAnchor(); + Pointer getStartSP(); @RawField - void setAnchor(JavaFrameAnchor anchor); + void setStartSP(Pointer sp); @RawField Pointer getEndSP(); @@ -72,17 +75,21 @@ public interface JavaStackWalk extends PointerBase { @RawField void setEndSP(Pointer sp); - // these fields are for diagnostics - @RawField - Pointer getStartSP(); + CodePointer getStartIP(); @RawField - void setStartSP(Pointer sp); + void setStartIP(CodePointer ip); @RawField - CodePointer getStartIP(); + JavaFrameAnchor getFrameAnchor(); @RawField - void setStartIP(CodePointer ip); + void setFrameAnchor(JavaFrameAnchor anchor); + + /* + * Fields for the current Java frame - co-located in the same struct. Note that this data is + * updated in-place when moving to a new frame. + */ + /* JavaFrame frame; */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index 9814ebcf0b97..087b7ae5b5bd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -24,10 +24,17 @@ */ package com.oracle.svm.core.stack; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CFunction.Transition; import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -35,217 +42,348 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.CodeInfoAccess; -import com.oracle.svm.core.code.CodeInfoQueryResult; import com.oracle.svm.core.code.CodeInfoTable; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; import com.oracle.svm.core.code.UntetheredCodeInfo; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.deopt.DeoptimizationSupport; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.jfr.JfrStackWalker; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.ContinuationSupport; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads.SafepointBehavior; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.core.common.NumUtil; + /** - * Provides methods to iterate each of the Java frames in a thread stack. It skips native frames, - * i.e., it only visits frames where {@link CodeInfoAccess#lookupTotalFrameSize Java frame - * information} is available. + * Provides methods to iterate over the physical Java stack frames of a thread (native stack frames + * are skipped, see {@link JavaFrameAnchor}). When starting a stack walk, it is possible to + * explicitly pass a stack pointer (SP) and an instruction pointer (IP) to specify where the stack + * walk should be started. Note that this first frame must always be a Java frame. *

* For most cases, the "walk*" methods that apply a {@link StackFrameVisitor} are the preferred way * to do stack walking. Use cases that are extremely performance sensitive, or cannot use a visitor - * approach, can use the various "init*" and "continue*" methods directly. + * approach, can use the various "initialize" and "advance" methods directly. + *

+ * The stack walking code must be uninterruptible so that it can be used during garbage collection + * and in situations where we are out of Java heap memory. It also must not have any static state so + * that multiple threads can walk their stacks concurrently. State is therefore stored in a + * stack-allocated {@link JavaStackWalk} structure. *

- * The stack walking code must be allocation free (so that it can be used during garbage collection) - * and not use static state (so that multiple threads can walk their stacks concurrently). State is - * therefore stored in a stack-allocated {@link JavaStackWalk} structure. + * If this implementation changes, other code that knows a lot about stack walking (e.g., + * {@link JfrStackWalker}) needs to be changed as well. + * + *

+ * Stack walking may skip frames of AOT or JIT-compiled code if the walked thread has C code on the + * stack that was called without a transition. In some situations, this can result in unexpected + * behavior. Note that signal handlers are also essentially C code that is called without a + * transition. Here is one example: + *

*/ public final class JavaStackWalker { - + @Platforms(Platform.HOSTED_ONLY.class) private JavaStackWalker() { } + @Fold + static int getJavaFrameOffset() { + return NumUtil.roundUp(SizeOf.get(JavaStackWalkImpl.class), ConfigurationValues.getTarget().wordSize); + } + + @Fold + public static int sizeOfJavaStackWalk() { + return getJavaFrameOffset() + SizeOf.get(JavaFrame.class); + } + /** - * Initialize a stack walk for the current thread. The given {@code walk} parameter should - * normally be allocated on the stack. - *

- * The stack walker is only valid while the stack being walked is stable and existent. - * - * @param walk the stack-allocated walk base pointer - * @param startSP the starting SP - * @param startIP the starting IP + * Returns information about the current physical Java frame. Note that this data is updated + * in-place when {@link #advance} is called. During a stack walk, it is therefore not possible + * to access the data of a previous frame. */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static JavaFrame getCurrentFrame(JavaStackWalk walk) { + return (JavaFrame) ((Pointer) walk).add(getJavaFrameOffset()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static Pointer getStartSP(JavaStackWalk walk) { + return cast(walk).getStartSP(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static Pointer getEndSP(JavaStackWalk walk) { + return cast(walk).getEndSP(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static JavaFrameAnchor getFrameAnchor(JavaStackWalk walk) { + return cast(walk).getFrameAnchor(); + } + @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) - public static void initWalk(JavaStackWalk walk, Pointer startSP, CodePointer startIP) { - initWalk(walk, startSP, WordFactory.nullPointer(), startIP, JavaFrameAnchors.getFrameAnchor()); + public static void initialize(JavaStackWalk walk, IsolateThread thread) { + initializeFromFrameAnchor(walk, thread, WordFactory.nullPointer()); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) - private static void initWalk(JavaStackWalk walk, Pointer startSP, Pointer endSP, CodePointer startIP, JavaFrameAnchor anchor) { - walk.setSP(startSP); - walk.setPossiblyStaleIP(startIP); - walk.setStartSP(startSP); - walk.setStartIP(startIP); - walk.setAnchor(anchor); - walk.setEndSP(endSP); - if (startIP.isNonNull()) { - // Storing the untethered object in a data structures requires that the caller and all - // places that use that value are uninterruptible as well. - walk.setIPCodeInfo(CodeInfoTable.lookupCodeInfo(startIP)); - } else { // will be read from the stack later - walk.setIPCodeInfo(WordFactory.nullPointer()); - } + public static void initialize(JavaStackWalk walk, IsolateThread thread, Pointer startSP) { + initWalk(walk, thread, startSP, WordFactory.nullPointer(), WordFactory.nullPointer(), JavaFrameAnchors.getFrameAnchor(thread)); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void initWalkStoredContinuation(JavaStackWalk walk, Pointer startSP, Pointer endSP, CodePointer startIP) { - /* - * StoredContinuations don't need a Java frame anchor because we pin the thread (i.e., - * yielding is not possible) if any native code is called. - */ - initWalk(walk, startSP, endSP, startIP, WordFactory.nullPointer()); + @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) + public static void initialize(JavaStackWalk walk, IsolateThread thread, Pointer startSP, Pointer endSP) { + initWalk(walk, thread, startSP, endSP, WordFactory.nullPointer(), JavaFrameAnchors.getFrameAnchor(thread)); } /** - * See {@link #initWalk(JavaStackWalk, Pointer, CodePointer)}, except that the instruction - * pointer will be read from the stack later on. + * This method should only be used rarely as it is usually not necessary (and potentially + * dangerous) to specify a {@code startIP} for the stack walk. */ - @Uninterruptible(reason = "Must be uninterruptible because it calls `initWalk`.") - public static void initWalk(JavaStackWalk walk, Pointer startSP) { - initWalk(walk, startSP, (CodePointer) WordFactory.nullPointer()); - assert walk.getIPCodeInfo().isNull() : "otherwise, the caller would have to be uninterruptible as well"; - } - - @Uninterruptible(reason = "Must be uninterruptible because it calls `initWalk`.") - public static void initWalk(JavaStackWalk walk, Pointer startSP, Pointer endSP) { - initWalk(walk, startSP); - walk.setEndSP(endSP); + @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) + public static void initialize(JavaStackWalk walk, IsolateThread thread, Pointer startSP, CodePointer startIP) { + initWalk(walk, thread, startSP, WordFactory.nullPointer(), startIP, JavaFrameAnchors.getFrameAnchor(thread)); } /** - * Initialize a stack walk for the given thread. The given {@code walk} parameter should - * normally be allocated on the stack. - * - * @param walk the stack-allocated walk base pointer - * @param thread the thread to examine + * This method should only be used rarely as it is usually not necessary (and potentially + * dangerous) to specify a {@code startIP} for the stack walk. */ @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) - public static boolean initWalk(JavaStackWalk walk, IsolateThread thread) { - assert thread.notEqual(CurrentIsolate.getCurrentThread()) : "Cannot walk the current stack with this method, it would miss all frames after the last frame anchor"; - assert VMOperation.isInProgressAtSafepoint() : "Walking the stack of another thread is only safe when that thread is stopped at a safepoint"; + public static void initialize(JavaStackWalk walk, IsolateThread thread, Pointer startSP, CodePointer startIP, JavaFrameAnchor anchor) { + initWalk(walk, thread, startSP, WordFactory.nullPointer(), startIP, anchor); + } - if (SafepointBehavior.isCrashedThread(thread)) { - /* Skip crashed threads because they may no longer have a stack. */ - return false; - } + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + public static void initialize(JavaStackWalk walk, StoredContinuation continuation) { + assert continuation != null; - JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); - boolean result = anchor.isNonNull(); - Pointer sp = WordFactory.nullPointer(); - CodePointer ip = WordFactory.nullPointer(); - if (result) { - sp = anchor.getLastJavaSP(); - ip = anchor.getLastJavaIP(); + CodePointer startIP = StoredContinuationAccess.getIP(continuation); + if (startIP.isNull()) { + /* StoredContinuation is uninitialized and therefore not walkable. */ + markAsNotWalkable(walk); + } else { + Pointer startSP = StoredContinuationAccess.getFramesStart(continuation); + Pointer endSP = StoredContinuationAccess.getFramesEnd(continuation); + initWalk0(walk, startSP, endSP, startIP, WordFactory.nullPointer()); } + } - walk.setSP(sp); - walk.setPossiblyStaleIP(ip); - walk.setStartSP(sp); - walk.setStartIP(ip); - walk.setAnchor(anchor); - walk.setEndSP(WordFactory.nullPointer()); - // Storing the untethered object in a data structures requires that the caller and all - // places that use that value are uninterruptible as well. - walk.setIPCodeInfo(CodeInfoTable.lookupCodeInfo(ip)); + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) + public static void initialize(JavaStackWalk walk, StoredContinuation continuation, CodePointer startIP) { + assert continuation != null; + assert startIP.isNonNull(); - return result; + Pointer startSP = StoredContinuationAccess.getFramesStart(continuation); + Pointer endSP = StoredContinuationAccess.getFramesEnd(continuation); + initWalk0(walk, startSP, endSP, startIP, WordFactory.nullPointer()); } - /** - * Continue a started stack walk. This method must only be called after {@link #initWalk} was - * called to start the walk. Once this method returns {@code false}, it will always return - * {@code false}. - * - * @param walk the initiated stack walk pointer - * @return {@code true} if there is another frame, or {@code false} if there are no more frames - * to iterate - */ - @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") - public static boolean continueWalk(JavaStackWalk walk, CodeInfo info) { - if (walk.getSP().isNull() || walk.getPossiblyStaleIP().isNull()) { - return false; + @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) + private static void initializeFromFrameAnchor(JavaStackWalk walk, IsolateThread thread, Pointer endSP) { + assert thread != CurrentIsolate.getCurrentThread() : "Walking the stack without specifying a start SP is only allowed when walking other threads"; + + JavaFrameAnchor frameAnchor = JavaFrameAnchors.getFrameAnchor(thread); + if (frameAnchor.isNull()) { + /* Threads that do not have a frame anchor at a safepoint are not walkable. */ + markAsNotWalkable(walk); + } else { + initWalk(walk, thread, frameAnchor.getLastJavaSP(), endSP, frameAnchor.getLastJavaIP(), frameAnchor.getPreviousAnchor()); } + } - Pointer sp = walk.getSP(); - SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); + @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) + private static void initWalk(JavaStackWalk walk, IsolateThread thread, Pointer startSP, Pointer endSP, CodePointer startIP, JavaFrameAnchor anchor) { + assert thread.isNonNull(); + assert thread == CurrentIsolate.getCurrentThread() || VMOperation.isInProgressAtSafepoint() : "Walking the stack of another thread is only safe when that thread is stopped at a safepoint"; + assert startSP.isNonNull(); - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); - if (deoptFrame == null) { - CodeInfoAccess.lookupCodeInfo(info, CodeInfoAccess.relativeIP(info, walk.getPossiblyStaleIP()), queryResult); + if (SafepointBehavior.isCrashedThread(thread)) { + /* Crashed threads may no longer have a stack. */ + markAsNotWalkable(walk); + } else { + initWalk0(walk, startSP, endSP, startIP, anchor); } + } - return continueWalk(walk, queryResult, deoptFrame); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void markAsNotWalkable(JavaStackWalk walk) { + initWalk0(walk, WordFactory.nullPointer(), WordFactory.nullPointer(), WordFactory.nullPointer(), WordFactory.nullPointer()); } - @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) - public static boolean continueWalk(JavaStackWalk walk, SimpleCodeInfoQueryResult queryResult, DeoptimizedFrame deoptFrame) { - boolean moreFrames; - Pointer sp = walk.getSP(); + @Uninterruptible(reason = "JavaStackWalk must not contain stale values when this method returns.", callerMustBe = true) + private static void initWalk0(JavaStackWalk walk, Pointer startSP, Pointer endSP, CodePointer startIP, JavaFrameAnchor anchor) { + JavaStackWalkImpl w = cast(walk); + w.setStarted(false); + w.setStartSP(startSP); + w.setEndSP(endSP); + w.setStartIP(startIP); + w.setFrameAnchor(anchor); + + JavaFrame frame = getCurrentFrame(walk); + JavaFrames.clearData(frame); + } - long encodedFrameSize; - if (deoptFrame != null) { - encodedFrameSize = deoptFrame.getSourceEncodedFrameSize(); - } else { - encodedFrameSize = queryResult.getEncodedFrameSize(); + @Uninterruptible(reason = "JavaStackWalk must not contain stale values when this method returns.", callerMustBe = true) + public static void updateStackPointer(JavaStackWalk walk, StoredContinuation continuation, int startSPOffset) { + JavaStackWalkImpl w = cast(walk); + Pointer newStartSP = StoredContinuationAccess.getFramesStart(continuation).add(startSPOffset); + long delta = newStartSP.rawValue() - w.getStartSP().rawValue(); + long newEndSP = w.getEndSP().rawValue() + delta; + + w.setStartSP(newStartSP); + w.setEndSP(WordFactory.pointer(newEndSP)); + + JavaFrame frame = getCurrentFrame(walk); + if (frame.getSP().isNonNull()) { + long newSP = frame.getSP().rawValue() + delta; + frame.setSP(WordFactory.pointer(newSP)); } + } - if (!CodeInfoQueryResult.isEntryPoint(encodedFrameSize)) { - long totalFrameSize = CodeInfoQueryResult.getTotalFrameSize(encodedFrameSize); + @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) + public static boolean advance(JavaStackWalk walk, IsolateThread thread) { + return advance(walk, thread, null); + } - /* Bump sp *up* over my frame. */ - sp = sp.add(WordFactory.unsigned(totalFrameSize)); - /* Read the return address to my caller. */ - CodePointer ip = FrameAccess.singleton().readReturnAddress(sp); + @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) + public static boolean advance(JavaStackWalk walk, StoredContinuation continuation) { + return advance(walk, WordFactory.nullPointer(), continuation); + } - walk.setSP(sp); - walk.setPossiblyStaleIP(ip); + @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) + private static boolean advance(JavaStackWalk walk, IsolateThread thread, StoredContinuation continuation) { + JavaStackWalkImpl w = cast(walk); + if (!w.getStarted()) { + return startStackWalk(w, thread, continuation); + } + return advance0(w, thread, continuation); + } - walk.setIPCodeInfo(CodeInfoTable.lookupCodeInfo(ip)); - moreFrames = true; + @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) + private static boolean startStackWalk(JavaStackWalkImpl walk, IsolateThread thread, StoredContinuation continuation) { + walk.setStarted(true); - } else { - /* Reached an entry point frame. */ + JavaFrame frame = getCurrentFrame(walk); + Pointer startSP = walk.getStartSP(); + Pointer endSP = walk.getEndSP(); + if (startSP.isNull() || endSP.isNonNull() && endSP.belowOrEqual(startSP)) { + /* The stack is not walkable or there are no frames to walk. */ + JavaFrames.clearData(frame); + return false; + } - JavaFrameAnchor anchor = walk.getAnchor(); - while (anchor.isNonNull() && anchor.getLastJavaSP().belowOrEqual(sp)) { - /* Skip anchors that are in parts of the stack we are not traversing. */ - anchor = anchor.getPreviousAnchor(); + /* Determine the actual start IP. */ + CodePointer startIP = walk.getStartIP(); + if (startIP.isNull()) { + JavaFrameAnchor anchor = skipUnnecessaryFrameAnchors(walk, startSP); + if (anchor.isNonNull() && startSP == anchor.getLastJavaSP()) { + startIP = anchor.getLastJavaIP(); + } else { + startIP = readReturnAddress(thread, continuation, startSP); } + walk.setStartIP(startIP); + } else { + assert CodeInfoTable.lookupCodeInfo(startIP).isNonNull(); + } - if (anchor.isNonNull()) { - /* We have more Java frames after a block of C frames. */ - assert anchor.getLastJavaSP().aboveThan(sp); - walk.setSP(anchor.getLastJavaSP()); - walk.setPossiblyStaleIP(anchor.getLastJavaIP()); - walk.setAnchor(anchor.getPreviousAnchor()); - walk.setIPCodeInfo(CodeInfoTable.lookupCodeInfo(anchor.getLastJavaIP())); - moreFrames = true; + /* It is guaranteed that we have at least one frame. */ + JavaFrames.setData(frame, startSP, startIP); + return true; + } - } else { - /* Really at the end of the stack, we are done with walking. */ - walk.setSP(WordFactory.nullPointer()); - walk.setPossiblyStaleIP(WordFactory.nullPointer()); - walk.setAnchor(WordFactory.nullPointer()); - walk.setIPCodeInfo(WordFactory.nullPointer()); - moreFrames = false; + @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) + private static boolean advance0(JavaStackWalkImpl walk, IsolateThread thread, StoredContinuation continuation) { + assert thread.isNull() != (continuation == null); + + JavaFrame frame = getCurrentFrame(walk); + Pointer sp = frame.getSP(); + if (sp.isNull()) { + /* No more frames to walk. */ + JavaFrames.clearData(frame); + return false; + } + + Pointer endSP = walk.getEndSP(); + sp = sp.add(JavaFrames.getTotalFrameSize(frame)); + if (JavaFrames.isEntryPoint(frame)) { + /* Use the frame anchor to skip the native frames. */ + JavaFrameAnchor anchor = skipUnnecessaryFrameAnchors(walk, sp); + if (anchor.isNonNull()) { + walk.setFrameAnchor(anchor.getPreviousAnchor()); + if (endSP.isNull() || endSP.aboveThan(anchor.getLastJavaSP())) { + JavaFrames.setData(frame, anchor.getLastJavaSP(), anchor.getLastJavaIP()); + return true; + } + } + } else { + /* Caller is a Java frame. */ + if (endSP.isNull() || endSP.aboveThan(sp)) { + CodePointer ip = readReturnAddress(thread, continuation, sp); + JavaFrames.setData(frame, sp, ip); + return true; } } - if (moreFrames && walk.getEndSP().isNonNull() && walk.getSP().aboveOrEqual(walk.getEndSP())) { - moreFrames = false; + /* No more frames. */ + JavaFrames.clearData(frame); + return false; + } + + /** + * Skip frame anchors that are not needed for this stack walk. This is necessary because: + *

+ */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static JavaFrameAnchor skipUnnecessaryFrameAnchors(JavaStackWalkImpl walk, Pointer sp) { + JavaFrameAnchor anchor = walk.getFrameAnchor(); + while (anchor.isNonNull() && anchor.getLastJavaSP().belowThan(sp)) { + anchor = anchor.getPreviousAnchor(); + } + walk.setFrameAnchor(anchor); + return anchor; + } + + @Uninterruptible(reason = "Stored continuation must not move.") + private static CodePointer readReturnAddress(IsolateThread thread, StoredContinuation continuation, Pointer startSP) { + if (ContinuationSupport.isSupported() && continuation != null) { + assert thread.isNull(); + return FrameAccess.singleton().readReturnAddress(continuation, startSP); + } else { + assert thread.isNonNull() && continuation == null; + return FrameAccess.singleton().readReturnAddress(thread, startSP); } - return moreFrames; } @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) @@ -257,7 +395,6 @@ public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePoi } log.newline(); throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); - } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") @@ -267,24 +404,26 @@ public static boolean walkCurrentThread(Pointer startSP, StackFrameVisitor visit @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") public static boolean walkCurrentThread(Pointer startSP, Pointer endSP, StackFrameVisitor visitor) { - return walkCurrentThread(startSP, endSP, FrameAccess.singleton().readReturnAddress(startSP), visitor, null); + assert startSP.isNonNull(); + return walkCurrentThread(startSP, endSP, WordFactory.nullPointer(), visitor, null); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") public static boolean walkCurrentThread(Pointer startSP, ParameterizedStackFrameVisitor visitor, Object data) { - return walkCurrentThread(startSP, WordFactory.nullPointer(), FrameAccess.singleton().readReturnAddress(startSP), visitor, data); + return walkCurrentThread(startSP, WordFactory.nullPointer(), WordFactory.nullPointer(), visitor, data); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static boolean walkCurrentThread(Pointer startSP, CodePointer startIP, ParameterizedStackFrameVisitor visitor) { return walkCurrentThread(startSP, WordFactory.nullPointer(), startIP, visitor, null); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") public static boolean walkCurrentThread(Pointer startSP, Pointer endSP, CodePointer startIP, ParameterizedStackFrameVisitor visitor, Object data) { - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - initWalk(walk, startSP, endSP, startIP, JavaFrameAnchors.getFrameAnchor()); - return doWalk(walk, visitor, data); + IsolateThread thread = CurrentIsolate.getCurrentThread(); + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + initWalk(walk, thread, startSP, endSP, startIP, JavaFrameAnchors.getFrameAnchor()); + return doWalk(walk, thread, visitor, data); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") @@ -299,56 +438,70 @@ public static boolean walkThread(IsolateThread thread, ParameterizedStackFrameVi @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") public static boolean walkThread(IsolateThread thread, Pointer endSP, ParameterizedStackFrameVisitor visitor, Object data) { - assert thread.isNonNull(); - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - if (initWalk(walk, thread)) { - walk.setEndSP(endSP); - return doWalk(walk, visitor, data); - } else { - return true; - } + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + initializeFromFrameAnchor(walk, thread, endSP); + return doWalk(walk, thread, visitor, data); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") - public static void walkThreadAtSafepoint(Pointer startSP, Pointer endSP, CodePointer startIP, StackFrameVisitor visitor) { - assert VMOperation.isInProgressAtSafepoint(); - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - initWalk(walk, startSP, endSP, startIP, JavaFrameAnchors.getFrameAnchor()); - doWalk(walk, visitor, null); + public static void walkThread(IsolateThread thread, Pointer startSP, Pointer endSP, CodePointer startIP, StackFrameVisitor visitor) { + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + initWalk(walk, thread, startSP, endSP, startIP, JavaFrameAnchors.getFrameAnchor(thread)); + doWalk(walk, thread, visitor, null); } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.", callerMustBe = true) - static boolean doWalk(JavaStackWalk walk, ParameterizedStackFrameVisitor visitor, Object data) { - while (true) { - UntetheredCodeInfo untetheredInfo = walk.getIPCodeInfo(); - if (untetheredInfo.isNull()) { - return callUnknownFrame(walk, visitor, data); + static boolean doWalk(JavaStackWalk walk, IsolateThread thread, ParameterizedStackFrameVisitor visitor, Object data) { + while (advance(walk, thread)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + Pointer sp = frame.getSP(); + CodePointer ip = frame.getIP(); + + if (JavaFrames.isUnknownFrame(frame)) { + return callUnknownFrameVisitor(sp, ip, visitor, data); } - Object tether = CodeInfoAccess.acquireTether(untetheredInfo); - try { - CodeInfo tetheredInfo = CodeInfoAccess.convert(untetheredInfo, tether); - // now the value in walk.getIPCodeInfo() can be passed to interruptible code - if (!callVisitor(walk, tetheredInfo, visitor, data)) { + DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(frame); + if (deoptimizedFrame != null) { + if (!callDeoptimizedFrameVisitor(sp, ip, deoptimizedFrame, visitor, data)) { return false; } - if (!continueWalk(walk, tetheredInfo)) { - return true; + } else { + UntetheredCodeInfo untetheredInfo = frame.getIPCodeInfo(); + Object tether = CodeInfoAccess.acquireTether(untetheredInfo); + try { + CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); + if (!callRegularFrameVisitor(sp, ip, info, visitor, data)) { + return false; + } + } finally { + CodeInfoAccess.releaseTether(untetheredInfo, tether); } - } finally { - CodeInfoAccess.releaseTether(untetheredInfo, tether); } } + + return true; } @Uninterruptible(reason = "CodeInfo in JavaStackWalk is currently null, and we are going to abort the stack walking.", calleeMustBe = false) - private static boolean callUnknownFrame(JavaStackWalk walk, ParameterizedStackFrameVisitor visitor, Object data) { - return visitor.unknownFrame(walk.getSP(), walk.getPossiblyStaleIP(), Deoptimizer.checkDeoptimized(walk.getSP()), data); + private static boolean callUnknownFrameVisitor(Pointer sp, CodePointer ip, ParameterizedStackFrameVisitor visitor, Object data) { + return visitor.unknownFrame(sp, ip, data); } @Uninterruptible(reason = "Wraps the now safe call to the possibly interruptible visitor.", callerMustBe = true, calleeMustBe = false) @RestrictHeapAccess(reason = "Whitelisted because some StackFrameVisitor implementations can allocate.", access = RestrictHeapAccess.Access.UNRESTRICTED) - public static boolean callVisitor(JavaStackWalk walk, CodeInfo info, ParameterizedStackFrameVisitor visitor, Object data) { - return visitor.visitFrame(walk.getSP(), walk.getPossiblyStaleIP(), info, Deoptimizer.checkDeoptimized(walk.getSP()), data); + private static boolean callRegularFrameVisitor(Pointer sp, CodePointer ip, CodeInfo info, ParameterizedStackFrameVisitor visitor, Object data) { + return visitor.visitRegularFrame(sp, ip, info, data); + } + + @Uninterruptible(reason = "Wraps the now safe call to the possibly interruptible visitor.", callerMustBe = true, calleeMustBe = false) + @RestrictHeapAccess(reason = "Whitelisted because some StackFrameVisitor implementations can allocate.", access = RestrictHeapAccess.Access.UNRESTRICTED) + private static boolean callDeoptimizedFrameVisitor(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, ParameterizedStackFrameVisitor visitor, Object data) { + return visitor.visitDeoptimizedFrame(sp, ip, deoptimizedFrame, data); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static JavaStackWalkImpl cast(JavaStackWalk walk) { + return (JavaStackWalkImpl) walk; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java index f4cea415987e..f24f6f4afa8d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.util.VMError.intentionallyUnimplemented; import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; @@ -283,7 +284,8 @@ private void checkDeoptimized() { } private VirtualFrame lookupVirtualFrame() { - DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(sp); + IsolateThread thread = CurrentIsolate.getCurrentThread(); + DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(thread, sp); if (deoptimizedFrame != null) { /* * Find the matching inlined frame, by skipping over the virtual frames that were @@ -301,9 +303,10 @@ private VirtualFrame lookupVirtualFrame() { @Override public void materializeVirtualObjects(boolean invalidateCode) { + IsolateThread thread = CurrentIsolate.getCurrentThread(); if (virtualFrame == null) { DeoptimizedFrame deoptimizedFrame = getDeoptimizer().deoptSourceFrame(ip, false); - assert deoptimizedFrame == Deoptimizer.checkDeoptimized(sp); + assert deoptimizedFrame == Deoptimizer.checkDeoptimized(thread, sp); } if (invalidateCode) { @@ -313,7 +316,7 @@ public void materializeVirtualObjects(boolean invalidateCode) { * a virtual object that was accessed via a local variable before would now have a * different value. */ - Deoptimizer.invalidateMethodOfFrame(sp, null); + Deoptimizer.invalidateMethodOfFrame(thread, sp, null); } /* We must be deoptimized now. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java index ad127c639cb0..fc0b37952fde 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.stack; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; @@ -37,6 +38,7 @@ import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.FrameInfoQueryResult; import com.oracle.svm.core.code.ImageCodeInfo; +import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.deopt.DeoptimizationSupport; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -134,9 +136,12 @@ protected void logFrame(Log log, Pointer sp, CodePointer ip, CodeInfo codeInfo, protected static void logFrameRaw(Log log, Pointer sp, CodePointer ip, CodeInfo codeInfo) { log.string("SP ").zhex(sp); log.string(" IP ").zhex(ip); + log.string(" size="); if (codeInfo.isNonNull()) { - long frameSize = CodeInfoAccess.lookupTotalFrameSize(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip)); - log.string(" size=").signed(frameSize, 4, Log.LEFT_ALIGN); + long frameSize = CodeInfoAccess.lookupTotalFrameSize(codeInfo, ip); + log.signed(frameSize, 4, Log.LEFT_ALIGN); + } else { + log.string("unknown"); } } } @@ -218,24 +223,30 @@ public static void printBacktrace() { } @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") - public static boolean printStacktrace(Pointer startSP, CodePointer startIP, Stage0StackFramePrintVisitor printVisitor, Log log) { - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - JavaStackWalker.initWalk(walk, startSP, startIP); - - JavaFrameAnchor anchor = walk.getAnchor(); - if (walk.getIPCodeInfo().isNull() && anchor.isNonNull()) { - logFrameAnchor(log, startSP, startIP); - walk.setSP(anchor.getLastJavaSP()); - walk.setPossiblyStaleIP(anchor.getLastJavaIP()); - walk.setIPCodeInfo(CodeInfoTable.lookupCodeInfo(anchor.getLastJavaIP())); + public static boolean printStacktrace(IsolateThread thread, Pointer initialSP, CodePointer initialIP, Stage0StackFramePrintVisitor printVisitor, Log log) { + Pointer sp = initialSP; + CodePointer ip = initialIP; + + /* Don't start the stack walk in a non-Java frame, even if the crash happened there. */ + UntetheredCodeInfo info = CodeInfoTable.lookupCodeInfo(ip); + if (info.isNull()) { + logFrame(log, sp, ip); + + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); + if (anchor.isNonNull()) { + sp = anchor.getLastJavaSP(); + ip = anchor.getLastJavaIP(); + } } - return JavaStackWalker.doWalk(walk, printVisitor, log); + JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, thread, sp, ip); + return JavaStackWalker.doWalk(walk, thread, printVisitor, log); } - @Uninterruptible(reason = "CodeInfo in JavaStackWalk is currently null, so printing to log is safe right now.", calleeMustBe = false) - private static void logFrameAnchor(Log log, Pointer startSP, CodePointer startIP) { - Stage0StackFramePrintVisitor.logFrameRaw(log, startSP, startIP, WordFactory.nullPointer()); + @Uninterruptible(reason = "IP is not within Java code, so there is no risk that it gets invalidated.", calleeMustBe = false) + private static void logFrame(Log log, Pointer sp, CodePointer ip) { + Stage0StackFramePrintVisitor.logFrameRaw(log, sp, ip, WordFactory.nullPointer()); log.string(" IP is not within Java code. Trying frame anchor of last Java frame instead.").newline(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 3e33f5c0feed..30341e48432b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -59,7 +59,6 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CEntryPointLiteral; import org.graalvm.nativeimage.c.function.CFunctionPointer; -import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.CCharPointer; @@ -70,7 +69,6 @@ import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; import com.oracle.svm.core.SubstrateOptions; @@ -878,7 +876,7 @@ static void visitCurrentStackFrames(Pointer callerSP, StackFrameVisitor visitor) } static StackTraceElement[] getStackTraceAtSafepoint(Thread thread, Pointer callerSP) { - assert thread != null && !isVirtual(thread); + assert thread != null && !isVirtual(thread) : "may only be called for platform or carrier threads"; Pointer carrierSP = getCarrierSPOrElse(thread, WordFactory.nullPointer()); IsolateThread isolateThread = getIsolateThread(thread); if (isolateThread == CurrentIsolate.getCurrentThread()) { @@ -889,11 +887,15 @@ static StackTraceElement[] getStackTraceAtSafepoint(Thread thread, Pointer calle */ return StackTraceUtils.getStackTrace(false, startSP, WordFactory.nullPointer()); } - if (carrierSP.isNonNull()) { // mounted virtual thread, skip its frames - CodePointer carrierIP = FrameAccess.singleton().readReturnAddress(carrierSP); - return StackTraceUtils.getThreadStackTraceAtSafepoint(carrierSP, WordFactory.nullPointer(), carrierIP); + if (carrierSP.isNonNull()) { + /* + * The given thread is a carrier thread and has a mounted virtual thread. Skip the + * frames of the virtual thread and only visit the stack frames that belong to the + * carrier thread. + */ + return StackTraceUtils.getStackTraceAtSafepoint(isolateThread, carrierSP, WordFactory.nullPointer()); } - return StackTraceUtils.getThreadStackTraceAtSafepoint(isolateThread, WordFactory.nullPointer()); + return StackTraceUtils.getThreadStackTraceAtSafepoint(isolateThread); } static Pointer getCarrierSPOrElse(Thread carrier, Pointer other) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index fd7bd5884c63..63c09cf6e19d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -156,7 +156,7 @@ public static VMThreads singleton() { private static final FastThreadLocalWord OSThreadIdTL = FastThreadLocalFactory.createWord("VMThreads.OSThreadIdTL"); public static final FastThreadLocalWord OSThreadHandleTL = FastThreadLocalFactory.createWord("VMThreads.OSThreadHandleTL"); public static final FastThreadLocalWord IsolateTL = FastThreadLocalFactory.createWord("VMThreads.IsolateTL"); - /** The highest stack address. */ + /** The highest stack address. 0 if not available on this platform. */ public static final FastThreadLocalWord StackBase = FastThreadLocalFactory.createWord("VMThreads.StackBase"); /** * The lowest stack address. Note that this value does not necessarily match the value that is From cfe876bd9c44cecc6314e52425f34e6dbade24e6 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 10 May 2024 09:46:09 +0200 Subject: [PATCH 2/7] Rename CodeInfoAccess.convert(...) to CodeInfoAccess.unsafeConvert(...). --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 2 +- .../src/com/oracle/svm/core/code/CodeInfoAccess.java | 4 ++-- .../src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 88d8c9d9a166..70f0417167a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -843,7 +843,7 @@ private static void walkStack(IsolateThread thread, JavaStackWalk walk, ObjectRe DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(frame); if (deoptFrame == null) { Pointer sp = frame.getSP(); - CodeInfo codeInfo = CodeInfoAccess.convert(frame.getIPCodeInfo()); + CodeInfo codeInfo = CodeInfoAccess.unsafeConvert(frame.getIPCodeInfo()); NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); long referenceMapIndex = frame.getReferenceMapIndex(); if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java index 164cf54dbc18..4e158b5ddf33 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java @@ -117,7 +117,7 @@ public static void releaseTetherUnsafe(@SuppressWarnings("unused") UntetheredCod @Uninterruptible(reason = "Should be called from the same method as acquireTether.", callerMustBe = true) public static CodeInfo convert(UntetheredCodeInfo untetheredInfo, Object tether) { assert UntetheredCodeInfoAccess.getTetherUnsafe(untetheredInfo) == null || UntetheredCodeInfoAccess.getTetherUnsafe(untetheredInfo) == tether; - return convert(untetheredInfo); + return unsafeConvert(untetheredInfo); } /** @@ -125,7 +125,7 @@ public static CodeInfo convert(UntetheredCodeInfo untetheredInfo, Object tether) * but with less verification. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static CodeInfo convert(UntetheredCodeInfo untetheredInfo) { + public static CodeInfo unsafeConvert(UntetheredCodeInfo untetheredInfo) { assert isValid(untetheredInfo); return (CodeInfo) untetheredInfo; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 2315624e1dac..cb6632bbc274 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -336,7 +336,7 @@ public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); if (untetheredInfo.isNonNull()) { /* We are during a GC, so no need for a tether. */ - CodeInfo info = CodeInfoAccess.convert(untetheredInfo); + CodeInfo info = CodeInfoAccess.unsafeConvert(untetheredInfo); callVisitor(visitor, info); } From b7d65858ebdbbbc4e36deed06e948699b2c5e050 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 10 May 2024 10:30:22 +0200 Subject: [PATCH 3/7] Renamed methods in StackTraceUtils. --- .../oracle/svm/core/jdk/StackTraceUtils.java | 22 ++++++++++--------- .../oracle/svm/core/thread/JavaThreads.java | 4 ++-- .../svm/core/thread/PlatformThreads.java | 6 ++--- .../com/oracle/svm/test/StackTraceTests.java | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java index 0e18ffd4a71e..662ea81621ee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java @@ -82,7 +82,7 @@ public class StackTraceUtils { * Captures at most {@link SubstrateOptions#maxJavaStackTraceDepth()} stack trace elements if * max depth > 0, or all if max depth <= 0. */ - public static StackTraceElement[] getStackTrace(boolean filterExceptions, Pointer startSP, Pointer endSP) { + public static StackTraceElement[] getCurrentThreadStackTrace(boolean filterExceptions, Pointer startSP, Pointer endSP) { BuildStackTraceVisitor visitor = new BuildStackTraceVisitor(filterExceptions, SubstrateOptions.maxJavaStackTraceDepth()); visitCurrentThreadStackFrames(startSP, endSP, visitor); return visitor.trace.toArray(NO_ELEMENTS); @@ -108,7 +108,11 @@ public static StackTraceElement[] getStackTraceAtSafepoint(Thread thread) { return JavaThreads.getStackTraceAtSafepoint(thread, readCallerStackPointer()); } - public static StackTraceElement[] getThreadStackTraceAtSafepoint(IsolateThread isolateThread, Pointer endSP) { + public static StackTraceElement[] getStackTraceAtSafepoint(IsolateThread isolateThread) { + return getStackTraceAtSafepoint(isolateThread, WordFactory.nullPointer()); + } + + public static StackTraceElement[] getStackTraceAtSafepoint(IsolateThread isolateThread, Pointer endSP) { assert VMOperation.isInProgressAtSafepoint(); if (isolateThread.isNull()) { // recently launched thread return NO_ELEMENTS; @@ -118,10 +122,10 @@ public static StackTraceElement[] getThreadStackTraceAtSafepoint(IsolateThread i return visitor.trace.toArray(NO_ELEMENTS); } - public static StackTraceElement[] getThreadStackTraceAtSafepoint(Pointer startSP, Pointer endSP, CodePointer startIP) { + public static StackTraceElement[] getStackTraceAtSafepoint(IsolateThread isolateThread, Pointer startSP, Pointer endSP) { assert VMOperation.isInProgressAtSafepoint(); BuildStackTraceVisitor visitor = new BuildStackTraceVisitor(false, SubstrateOptions.maxJavaStackTraceDepth()); - JavaStackWalker.walkThreadAtSafepoint(startSP, endSP, startIP, visitor); + JavaStackWalker.walkThread(isolateThread, startSP, endSP, WordFactory.nullPointer(), visitor); return visitor.trace.toArray(NO_ELEMENTS); } @@ -209,10 +213,8 @@ public static boolean shouldShowFrame(Class clazz, String methodName, boolean } } - if (clazz == Target_jdk_internal_vm_Continuation.class) { - if (UninterruptibleUtils.String.startsWith(methodName, "enter") || UninterruptibleUtils.String.startsWith(methodName, "yield")) { - return false; - } + if (clazz == Target_jdk_internal_vm_Continuation.class && (UninterruptibleUtils.String.startsWith(methodName, "enter") || UninterruptibleUtils.String.startsWith(methodName, "yield"))) { + return false; } return true; @@ -710,7 +712,7 @@ class GetClassContextVisitor extends JavaStackFrameVisitor { } @Override - public boolean visitFrame(final FrameInfoQueryResult frameInfo) { + public boolean visitFrame(FrameInfoQueryResult frameInfo) { if (skip > 0) { skip--; } else if (StackTraceUtils.shouldShowFrame(frameInfo, true, false, false)) { @@ -762,7 +764,7 @@ class StackAccessControlContextVisitor extends JavaStackFrameVisitor { } @Override - public boolean visitFrame(final FrameInfoQueryResult frameInfo) { + public boolean visitFrame(FrameInfoQueryResult frameInfo) { if (!StackTraceUtils.shouldShowFrame(frameInfo, true, false, false)) { return true; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java index a3a5158ce61e..7213c8e1faf5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java @@ -221,10 +221,10 @@ private static StackTraceElement[] getMountedVirtualThreadStackTrace(boolean fil return null; } if (carrier == PlatformThreads.currentThread.get()) { - return StackTraceUtils.getStackTrace(filterExceptions, callerSP, endSP); + return StackTraceUtils.getCurrentThreadStackTrace(filterExceptions, callerSP, endSP); } assert VMOperation.isInProgressAtSafepoint(); - return StackTraceUtils.getThreadStackTraceAtSafepoint(PlatformThreads.getIsolateThread(carrier), endSP); + return StackTraceUtils.getStackTraceAtSafepoint(PlatformThreads.getIsolateThread(carrier), endSP); } public static StackTraceElement[] getStackTraceAtSafepoint(Thread thread, Pointer callerSP) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 30341e48432b..02b739935554 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -863,7 +863,7 @@ static StackTraceElement[] getStackTrace(boolean filterExceptions, Thread thread assert !isVirtual(thread); if (thread != null && thread == currentThread.get()) { Pointer startSP = getCarrierSPOrElse(thread, callerSP); - return StackTraceUtils.getStackTrace(filterExceptions, startSP, WordFactory.nullPointer()); + return StackTraceUtils.getCurrentThreadStackTrace(filterExceptions, startSP, WordFactory.nullPointer()); } assert !filterExceptions : "exception stack traces can be taken only for the current thread"; return StackTraceUtils.asyncGetStackTrace(thread); @@ -885,7 +885,7 @@ static StackTraceElement[] getStackTraceAtSafepoint(Thread thread, Pointer calle * Internal frames from the VMOperation handling show up in the stack traces, but we are * OK with that. */ - return StackTraceUtils.getStackTrace(false, startSP, WordFactory.nullPointer()); + return StackTraceUtils.getCurrentThreadStackTrace(false, startSP, WordFactory.nullPointer()); } if (carrierSP.isNonNull()) { /* @@ -895,7 +895,7 @@ static StackTraceElement[] getStackTraceAtSafepoint(Thread thread, Pointer calle */ return StackTraceUtils.getStackTraceAtSafepoint(isolateThread, carrierSP, WordFactory.nullPointer()); } - return StackTraceUtils.getThreadStackTraceAtSafepoint(isolateThread); + return StackTraceUtils.getStackTraceAtSafepoint(isolateThread); } static Pointer getCarrierSPOrElse(Thread carrier, Pointer other) { diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/StackTraceTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/StackTraceTests.java index 6e318f1d55a0..048f1efbff8d 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/StackTraceTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/StackTraceTests.java @@ -66,7 +66,7 @@ public static void c(Type type) { assertSame(B.class, callerClass); } if (type == Type.GET_STACKTRACE) { - StackTraceElement[] stackTrace = StackTraceUtils.getStackTrace(true, KnownIntrinsics.readCallerStackPointer(), WordFactory.nullPointer()); + StackTraceElement[] stackTrace = StackTraceUtils.getCurrentThreadStackTrace(true, KnownIntrinsics.readCallerStackPointer(), WordFactory.nullPointer()); assertTrue(stackTrace.length > 0); assertSame(B.class.getName(), stackTrace[0].getClassName()); assertSame(A.class.getName(), stackTrace[1].getClassName()); From 631f03750b22bc754f7c019bdde894f362e9579c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 10 May 2024 10:56:02 +0200 Subject: [PATCH 4/7] Changed the stack walking visitors so that they have different methods for regular and deoptimized frames. --- .../svm/core/genscavenge/PathExhibitor.java | 25 +++++----- .../svm/core/genscavenge/StackVerifier.java | 11 ++++- .../oracle/svm/core/code/CodeInfoTable.java | 11 +---- .../svm/core/code/RuntimeCodeCache.java | 7 ++- .../oracle/svm/core/deopt/Deoptimizer.java | 10 +++- .../svm/core/graal/snippets/DeoptTester.java | 8 +++- .../core/heap/StoredContinuationAccess.java | 7 ++- .../svm/core/heap/dump/HeapDumpWriter.java | 46 ++++++++++--------- .../oracle/svm/core/jdk/StackTraceUtils.java | 28 ++++++----- .../core/sampler/SamplingStackVisitor.java | 13 ++++-- .../svm/core/stack/JavaStackFrameVisitor.java | 26 ++++++----- .../svm/core/stack/JavaStackWalker.java | 6 +-- .../stack/ParameterizedStackFrameVisitor.java | 19 +++++--- .../svm/core/stack/StackFrameVisitor.java | 19 +++++--- .../stack/SubstrateStackIntrospection.java | 15 ++++-- .../svm/core/stack/ThreadStackPrinter.java | 29 ++++++------ 16 files changed, 166 insertions(+), 114 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java index d016de177695..ad6321a6ad84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.code.CodeInfo; @@ -272,23 +271,28 @@ void reset() { @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.") - public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { - frameSlotVisitor.initialize(ip, deoptimizedFrame, target, result); - return CodeInfoTable.visitObjectReferences(sp, ip, codeInfo, deoptimizedFrame, frameSlotVisitor); + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { + frameSlotVisitor.initialize(ip, target, result); + return CodeInfoTable.visitObjectReferences(sp, ip, codeInfo, frameSlotVisitor); + } + + @Override + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + /* Support for deoptimized frames is not implemented at the moment. */ + return true; } } private static class FrameSlotVisitor extends AbstractVisitor implements ObjectReferenceVisitor { private CodePointer ip; - private DeoptimizedFrame deoptFrame; FrameSlotVisitor() { } - void initialize(CodePointer ipArg, DeoptimizedFrame deoptFrameArg, TargetMatcher targetMatcher, PathEdge edge) { + void initialize(CodePointer ipArg, TargetMatcher targetMatcher, PathEdge edge) { super.initialize(targetMatcher, edge); ip = ipArg; - deoptFrame = deoptFrameArg; } @Override @@ -300,7 +304,7 @@ public boolean visitObjectReference(Pointer stackSlot, boolean compressed, Objec Pointer referentPointer = ReferenceAccess.singleton().readObjectAsUntrackedPointer(stackSlot, compressed); trace.string(" referentPointer: ").zhex(referentPointer); if (target.matches(referentPointer.toObject())) { - result.fill(new StackElement(stackSlot, ip, deoptFrame), new LeafElement(referentPointer.toObject())); + result.fill(new StackElement(stackSlot, ip), new LeafElement(referentPointer.toObject())); return false; } return true; @@ -434,12 +438,10 @@ public Log toLog(Log log) { public static class StackElement extends PathElement { private final Pointer stackSlot; private final CodePointer ip; - private final CodePointer deoptSourcePC; private final Pointer slotValue; - StackElement(Pointer stackSlot, CodePointer ip, DeoptimizedFrame deoptFrame) { + StackElement(Pointer stackSlot, CodePointer ip) { this.stackSlot = stackSlot; - this.deoptSourcePC = deoptFrame != null ? deoptFrame.getSourcePC() : WordFactory.nullPointer(); this.ip = ip; this.slotValue = stackSlot.readWord(0); } @@ -454,7 +456,6 @@ public Object getObject() { public Log toLog(Log log) { log.string("[stack:"); log.string(" slot: ").zhex(stackSlot); - log.string(" deoptSourcePC: ").zhex(deoptSourcePC); log.string(" ip: ").zhex(ip); log.string(" value: ").zhex(slotValue); log.string("]"); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java index 31a1ea09f560..63642c13846e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java @@ -86,12 +86,19 @@ public void initialize() { @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while verifying the stack.") - public boolean visitFrame(Pointer currentSP, CodePointer currentIP, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { + public boolean visitRegularFrame(Pointer currentSP, CodePointer currentIP, CodeInfo codeInfo) { verifyFrameReferencesVisitor.initialize(); - CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, deoptimizedFrame, verifyFrameReferencesVisitor); + CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, verifyFrameReferencesVisitor); result &= verifyFrameReferencesVisitor.result; return true; } + + @Override + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while verifying the stack.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + /* Nothing to do. */ + return true; + } } private static class VerifyFrameReferencesVisitor implements ObjectReferenceVisitor { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index 7d23ff9f8f81..868164369398 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -38,7 +38,6 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; -import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; @@ -138,17 +137,9 @@ public static CodeInfoQueryResult lookupDeoptimizationEntrypoint(CodeInfo info, return result; } - public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo info, DeoptimizedFrame deoptimizedFrame, ObjectReferenceVisitor visitor) { + public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo info, ObjectReferenceVisitor visitor) { counters().visitObjectReferencesCount.inc(); - if (deoptimizedFrame != null) { - /* - * It is a deoptimized frame. The DeoptimizedFrame object is stored in the frame, but it - * is pinned so we do not have to do anything. - */ - return true; - } - /* * NOTE: if this code does not execute in a VM operation, it is possible for the visited * frame to be deoptimized concurrently, and that one of the references is overwritten with diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java index 6d2af1753d59..700fd769026e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java @@ -308,10 +308,15 @@ public boolean verify(CodeInfo info) { } @Override - public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo currentCodeInfo, DeoptimizedFrame deoptimizedFrame) { + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo currentCodeInfo) { assert currentCodeInfo != codeInfoToCheck : currentCodeInfo.rawValue(); return true; } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + return true; + } } /** This is the interface that clients have to implement. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java index daa79bf0360e..c11686150120 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java @@ -352,15 +352,21 @@ private static void deoptimizeInRangeOperation(CodePointer fromIp, CodePointer t private static StackFrameVisitor getStackFrameVisitor(Pointer fromIp, Pointer toIp, boolean deoptAll, IsolateThread targetThread) { return new StackFrameVisitor() { @Override - public boolean visitFrame(Pointer frameSp, CodePointer frameIp, CodeInfo codeInfo, DeoptimizedFrame deoptFrame) { + public boolean visitRegularFrame(Pointer frameSp, CodePointer frameIp, CodeInfo codeInfo) { Pointer ip = (Pointer) frameIp; - if (deoptFrame == null && ((ip.aboveOrEqual(fromIp) && ip.belowThan(toIp)) || deoptAll)) { + if ((ip.aboveOrEqual(fromIp) && ip.belowThan(toIp)) || deoptAll) { CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, frameIp); Deoptimizer deoptimizer = new Deoptimizer(frameSp, queryResult, targetThread); deoptimizer.deoptSourceFrame(frameIp, deoptAll); } return true; } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + /* Nothing to do. */ + return true; + } }; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java index eaa0a4fd3fc0..ba2d359cb394 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTester.java @@ -79,10 +79,16 @@ public static class Options { @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Only deals with IPs, not Objects.") - public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { handledPCs.add(ip.rawValue()); return true; } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + /* Nothing to do. */ + return true; + } }; @Fold diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index dd0858fd6dfd..fcfe1f74e2f5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -256,7 +256,7 @@ private static final class PreemptVisitor extends StackFrameVisitor { } @Override - protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { + protected boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { if (sp.aboveOrEqual(endSP)) { return false; } @@ -285,5 +285,10 @@ protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Deop return true; } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + throw VMError.shouldNotReachHere("Continuations can't contain JIT compiled code."); + } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java index 6d62eb98e078..2227c665f001 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java @@ -1193,30 +1193,32 @@ public int getWrittenFrames() { @Override @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") - protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { - if (deoptimizedFrame != null) { - markAsGCRoot(deoptimizedFrame); + protected boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { + /* + * All references that are on the stack need to be marked as GC roots. Our information + * is not necessarily precise enough to identify the exact Java-level stack frame to + * which a reference belongs. Therefore, we just dump the data in a way that it gets + * associated with the deepest inlined Java-level stack frame of each compilation unit. + */ + markStackValuesAsGCRoots(sp, ip, codeInfo); - for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { - visitFrame(frame.getFrameInfo()); - nextFrameId++; - } - } else { - /* - * All references that are on the stack need to be marked as GC roots. Our - * information is not necessarily precise enough to identify the exact Java-level - * stack frame to which a reference belongs. Therefore, we just dump the data in a - * way that it gets associated with the deepest inlined Java-level stack frame of - * each compilation unit. - */ - markStackValuesAsGCRoots(sp, ip, codeInfo); + frameInfoCursor.initialize(codeInfo, ip, true); + while (frameInfoCursor.advance()) { + FrameInfoQueryResult frame = frameInfoCursor.get(); + visitFrame(frame); + nextFrameId++; + } + return true; + } - frameInfoCursor.initialize(codeInfo, ip, true); - while (frameInfoCursor.advance()) { - FrameInfoQueryResult frame = frameInfoCursor.get(); - visitFrame(frame); - nextFrameId++; - } + @Override + @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + markAsGCRoot(deoptimizedFrame); + + for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { + visitFrame(frame.getFrameInfo()); + nextFrameId++; } return true; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java index 662ea81621ee..b9717145d83f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/StackTraceUtils.java @@ -386,23 +386,29 @@ private static int computeNativeLimit() { } @Override - protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { - if (deoptimizedFrame != null) { - for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { - FrameInfoQueryResult frameInfo = frame.getFrameInfo(); - if (!visitFrameInfo(frameInfo)) { - return false; - } - } - } else if (!CodeInfoTable.isInAOTImageCode(ip)) { + protected boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { + if (CodeInfoTable.isInAOTImageCode(ip)) { + visitAOTFrame(ip); + } else { CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, ip); + assert queryResult != null; + for (FrameInfoQueryResult frameInfo = queryResult.getFrameInfo(); frameInfo != null; frameInfo = frameInfo.getCaller()) { if (!visitFrameInfo(frameInfo)) { return false; } } - } else { - visitAOTFrame(ip); + } + return numFrames != limit; + } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { + FrameInfoQueryResult frameInfo = frame.getFrameInfo(); + if (!visitFrameInfo(frameInfo)) { + return false; + } } return numFrames != limit; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java index 553123082b0e..5f888f0ee016 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplingStackVisitor.java @@ -32,12 +32,13 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; +import com.oracle.svm.core.util.VMError; class SamplingStackVisitor extends ParameterizedStackFrameVisitor { - @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate within the safepoint sampler.") @Override - protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Object data) { + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate within the safepoint sampler.") + protected boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { SamplingStackVisitor.StackTrace stackTrace = (SamplingStackVisitor.StackTrace) data; if (stackTrace.num < stackTrace.buffer.length) { stackTrace.buffer[stackTrace.num++] = ip.rawValue(); @@ -49,7 +50,13 @@ protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Deop } @Override - protected boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data) { + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate within the safepoint sampler.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + throw VMError.shouldNotReachHere("Sampling is not supported if JIT compilation is enabled."); + } + + @Override + protected boolean unknownFrame(Pointer sp, CodePointer ip, Object data) { throw JavaStackWalker.fatalErrorUnknownFrameEncountered(sp, ip); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackFrameVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackFrameVisitor.java index a5351e77717c..8a86decb7aee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackFrameVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackFrameVisitor.java @@ -36,19 +36,21 @@ public abstract class JavaStackFrameVisitor extends StackFrameVisitor { @Override - public final boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { - if (deoptimizedFrame != null) { - for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { - if (!visitFrame(frame.getFrameInfo())) { - return false; - } + public final boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { + CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, ip); + for (FrameInfoQueryResult frameInfo = queryResult.getFrameInfo(); frameInfo != null; frameInfo = frameInfo.getCaller()) { + if (!visitFrame(frameInfo)) { + return false; } - } else { - CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, ip); - for (FrameInfoQueryResult frameInfo = queryResult.getFrameInfo(); frameInfo != null; frameInfo = frameInfo.getCaller()) { - if (!visitFrame(frameInfo)) { - return false; - } + } + return true; + } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + for (DeoptimizedFrame.VirtualFrame frame = deoptimizedFrame.getTopFrame(); frame != null; frame = frame.getCaller()) { + if (!visitFrame(frame.getFrameInfo())) { + return false; } } return true; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index 087b7ae5b5bd..7845f3cd5886 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -45,7 +45,6 @@ import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.UntetheredCodeInfo; import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.deopt.DeoptimizationSupport; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -387,12 +386,9 @@ private static CodePointer readReturnAddress(IsolateThread thread, StoredContinu } @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) - public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame) { + public static RuntimeException fatalErrorUnknownFrameEncountered(Pointer sp, CodePointer ip) { Log log = Log.log().string("Stack walk must walk only frames of known code:"); log.string(" sp=").zhex(sp).string(" ip=").zhex(ip); - if (DeoptimizationSupport.enabled()) { - log.string(" deoptFrame=").object(deoptFrame); - } log.newline(); throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java index 412ecda216cc..73f49219019e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java @@ -50,12 +50,21 @@ public abstract class ParameterizedStackFrameVisitor { * @param sp The stack pointer of the frame being visited. * @param ip The instruction pointer of the frame being visited. * @param codeInfo Information on the code at the IP, for use with {@link CodeInfoAccess}. - * @param deoptimizedFrame The information about a deoptimized frame, or {@code null} if the - * frame is not deoptimized. * @param data An arbitrary data value passed through the stack walker. * @return true if visiting should continue, false otherwise. */ - protected abstract boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Object data); + protected abstract boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data); + + /** + * Called for each deoptimized frame that is visited. + * + * @param originalSP The stack pointer to the physical (already invalidated) stack frame. + * @param deoptStubIP The instruction pointer for the deopt stub. + * @param deoptimizedFrame The deoptimized frame. + * @param data An arbitrary data value passed through the stack walker. + * @return true if visiting should continue, false otherwise. + */ + protected abstract boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data); /** * Called when no {@link CodeInfo frame metadata} can be found for a frame. That usually means @@ -65,11 +74,9 @@ public abstract class ParameterizedStackFrameVisitor { * * @param sp The stack pointer of the frame being visited. * @param ip The instruction pointer of the frame being visited. - * @param deoptimizedFrame The information about a deoptimized frame, or {@code null} if the - * frame is not deoptimized. * @param data An arbitrary data value passed through the stack walker. * @return The value returned to the caller of stack walking. Note that walking of the thread is * always aborted, regardless of the return value. */ - protected abstract boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data); + protected abstract boolean unknownFrame(Pointer sp, CodePointer ip, Object data); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/StackFrameVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/StackFrameVisitor.java index 5efc38fc8195..eabc782bef9c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/StackFrameVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/StackFrameVisitor.java @@ -45,19 +45,24 @@ public abstract class StackFrameVisitor extends ParameterizedStackFrameVisitor { * @param sp The stack pointer of the frame being visited. * @param ip The instruction pointer of the frame being visited. * @param codeInfo Information on the code at the IP, for use with {@link CodeInfoAccess}. - * @param deoptimizedFrame The information about a deoptimized frame, or {@code null} if the - * frame is not deoptimized. * @return true if visiting should continue, false otherwise. */ - protected abstract boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame); + protected abstract boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo); @Override - protected final boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Object data) { - return visitFrame(sp, ip, codeInfo, deoptimizedFrame); + protected final boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + return visitRegularFrame(sp, ip, codeInfo); + } + + protected abstract boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame); + + @Override + protected final boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + return visitDeoptimizedFrame(originalSP, deoptStubIP, deoptimizedFrame); } @Override - protected final boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptimizedFrame); + protected final boolean unknownFrame(Pointer sp, CodePointer ip, Object data) { + throw JavaStackWalker.fatalErrorUnknownFrameEncountered(sp, ip); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java index f24f6f4afa8d..507d478d44cf 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/SubstrateStackIntrospection.java @@ -99,7 +99,17 @@ class PhysicalStackFrameVisitor extends StackFrameVisitor { } @Override - public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { + return visitFrame(sp, ip, codeInfo, null); + } + + @Override + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { + CodeInfo imageCodeInfo = CodeInfoTable.lookupImageCodeInfo(deoptStubIP); + return visitFrame(originalSP, deoptStubIP, imageCodeInfo, deoptimizedFrame); + } + + private boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) { VirtualFrame virtualFrame = null; CodeInfoQueryResult info = null; FrameInfoQueryResult deoptInfo = null; @@ -160,7 +170,6 @@ public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Deoptim } while (virtualFrame != null || deoptInfo != null); return true; - } private static boolean matchesDeoptAddress(CodePointer ip, ResolvedJavaMethod[] methods) { @@ -181,7 +190,7 @@ class SubstrateInspectedFrame implements InspectedFrame { private final Pointer sp; private final CodePointer ip; protected VirtualFrame virtualFrame; - private CodeInfoQueryResult codeInfo; + private final CodeInfoQueryResult codeInfo; private FrameInfoQueryResult frameInfo; private final int virtualFrameIndex; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java index fc0b37952fde..79ac3a36ae50 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java @@ -30,7 +30,6 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.CodeInfoAccess; @@ -101,10 +100,20 @@ public Stage0StackFramePrintVisitor reset() { return this; } + @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Provide allocation-free StackFrameVisitor") + protected final boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + return visitFrame(sp, ip, codeInfo, null, (Log) data); + } + @Override - protected final boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptFrame, Object data) { - Log log = (Log) data; + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Provide allocation-free StackFrameVisitor") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptFrame, Object data) { + CodeInfo imageCodeInfo = CodeInfoTable.lookupImageCodeInfo(deoptStubIP); + return visitFrame(originalSP, deoptStubIP, imageCodeInfo, deoptFrame, (Log) data); + } + + private boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptFrame, Log log) { if (printedFrames >= MAX_STACK_FRAMES_PER_THREAD_TO_PRINT) { log.string("... (truncated)").newline(); return false; @@ -116,12 +125,9 @@ protected final boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo } @Override - protected final boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data) { + protected final boolean unknownFrame(Pointer sp, CodePointer ip, Object data) { Log log = (Log) data; logFrameRaw(log, sp, ip, WordFactory.nullPointer()); - if (DeoptimizationSupport.enabled()) { - log.string(" deoptFrame=").object(deoptimizedFrame); - } log.string(" IP is not within Java code. Aborting stack trace printing.").newline(); printedFrames++; return false; @@ -213,15 +219,6 @@ private static char getFrameIdentifier(CodeInfo codeInfo, DeoptimizedFrame deopt } } - /** - * Walk the stack printing each frame. - */ - @NeverInline("debugger breakpoint") - @Uninterruptible(reason = "Called from uninterruptible code.") - public static void printBacktrace() { - // Only used as a debugger breakpoint - } - @Uninterruptible(reason = "Prevent deoptimization of stack frames while in this method.") public static boolean printStacktrace(IsolateThread thread, Pointer initialSP, CodePointer initialIP, Stage0StackFramePrintVisitor printVisitor, Log log) { Pointer sp = initialSP; From fecc133925c6edc4289991abd70d6ea26df04747 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 10 May 2024 11:50:12 +0200 Subject: [PATCH 5/7] Fixes and changes related to the JFR execution sampler. --- .../oracle/svm/core/jfr/JfrBufferAccess.java | 4 +- .../com/oracle/svm/core/jfr/JfrFeature.java | 2 - .../oracle/svm/core/jfr/JfrGlobalMemory.java | 8 +- .../svm/core/jfr/JfrStackTraceRepository.java | 40 +- .../oracle/svm/core/jfr/JfrStackWalker.java | 385 ++++++++++++++++++ .../oracle/svm/core/jfr/JfrThreadLocal.java | 24 +- .../com/oracle/svm/core/jfr/SubstrateJVM.java | 11 +- .../sampler/AbstractJfrExecutionSampler.java | 172 +------- .../JfrRecurringCallbackExecutionSampler.java | 2 +- .../svm/core/sampler/SamplerBufferAccess.java | 9 +- .../svm/core/sampler/SamplerBufferPool.java | 6 +- .../core/sampler/SamplerSampleWriterData.java | 8 +- .../SamplerSampleWriterDataAccess.java | 4 +- .../core/sampler/SamplerStackWalkVisitor.java | 99 ----- 14 files changed, 447 insertions(+), 327 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java index 8c3a4389bbf8..cad4e1d5f74b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java @@ -55,8 +55,8 @@ public static UnsignedWord getHeaderSize() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBuffer allocate(JfrBufferType bufferType) { - long dataSize = SubstrateJVM.getThreadLocal().getThreadLocalBufferSize(); - return allocate(WordFactory.unsigned(dataSize), bufferType); + UnsignedWord dataSize = SubstrateJVM.getThreadLocal().getThreadLocalBufferSize(); + return allocate(dataSize, bufferType); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrFeature.java index 0715c7c7f45c..fccc3376c54e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrFeature.java @@ -41,7 +41,6 @@ import com.oracle.svm.core.jfr.traceid.JfrTraceIdMap; import com.oracle.svm.core.sampler.SamplerJfrStackTraceSerializer; import com.oracle.svm.core.sampler.SamplerStackTraceSerializer; -import com.oracle.svm.core.sampler.SamplerStackWalkVisitor; import com.oracle.svm.core.sampler.SamplerStatistics; import com.oracle.svm.core.thread.ThreadListenerSupport; import com.oracle.svm.core.thread.ThreadListenerSupportFeature; @@ -160,7 +159,6 @@ public void afterRegistration(AfterRegistrationAccess access) { ImageSingletons.add(JfrTraceIdMap.class, new JfrTraceIdMap()); ImageSingletons.add(JfrTraceIdEpoch.class, new JfrTraceIdEpoch()); ImageSingletons.add(JfrGCNames.class, new JfrGCNames()); - ImageSingletons.add(SamplerStackWalkVisitor.class, new SamplerStackWalkVisitor()); ImageSingletons.add(JfrExecutionSamplerSupported.class, new JfrExecutionSamplerSupported()); ImageSingletons.add(SamplerStackTraceSerializer.class, new SamplerJfrStackTraceSerializer()); ImageSingletons.add(SamplerStatistics.class, new SamplerStatistics()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrGlobalMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrGlobalMemory.java index 3a8e944d58da..249ba598e0a8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrGlobalMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrGlobalMemory.java @@ -45,19 +45,19 @@ public class JfrGlobalMemory { private static final int PROMOTION_RETRY_COUNT = 100; private final JfrBufferList buffers; - private long bufferSize; + private UnsignedWord bufferSize; @Platforms(Platform.HOSTED_ONLY.class) public JfrGlobalMemory() { this.buffers = new JfrBufferList(); } - public void initialize(long globalBufferSize, long globalBufferCount) { + public void initialize(UnsignedWord globalBufferSize, long globalBufferCount) { this.bufferSize = globalBufferSize; /* Allocate all buffers eagerly. */ for (int i = 0; i < globalBufferCount; i++) { - JfrBuffer buffer = JfrBufferAccess.allocate(WordFactory.unsigned(bufferSize), JfrBufferType.GLOBAL_MEMORY); + JfrBuffer buffer = JfrBufferAccess.allocate(bufferSize, JfrBufferType.GLOBAL_MEMORY); if (buffer.isNull()) { throw new OutOfMemoryError("Could not allocate JFR buffer."); } @@ -152,7 +152,7 @@ public boolean write(JfrBuffer buffer, UnsignedWord unflushedSize, boolean flush @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") private JfrBufferNode tryAcquirePromotionBuffer(UnsignedWord size) { - assert size.belowOrEqual(WordFactory.unsigned(bufferSize)); + assert size.belowOrEqual(bufferSize); for (int retry = 0; retry < PROMOTION_RETRY_COUNT; retry++) { JfrBufferNode node = buffers.getHead(); while (node.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java index c8b98185ca65..6db818c4cfe3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java @@ -24,7 +24,7 @@ */ package com.oracle.svm.core.jfr; -import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; @@ -55,9 +55,8 @@ import com.oracle.svm.core.sampler.SamplerSampleWriter; import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; -import com.oracle.svm.core.sampler.SamplerStackWalkVisitor; import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.util.VMError; /** * Repository that collects all metadata about stacktraces. @@ -109,23 +108,28 @@ public long getStackTraceId(int skipCount) { */ JfrExecutionSampler.singleton().preventSamplingInCurrentThread(); try { - /* Try to walk the stack. */ SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class); - if (SamplerSampleWriterDataAccess.initialize(data, skipCount, true)) { - JfrThreadLocal.setSamplerWriterData(data); - try { - SamplerSampleWriter.begin(data); - Pointer sp = KnownIntrinsics.readCallerStackPointer(); - CodePointer ip = FrameAccess.singleton().readReturnAddress(sp); - SamplerStackWalkVisitor visitor = ImageSingletons.lookup(SamplerStackWalkVisitor.class); - if (JavaStackWalker.walkCurrentThread(sp, ip, visitor) || data.getTruncated()) { - return storeDeduplicatedStackTrace(data); - } - } finally { - JfrThreadLocal.setSamplerWriterData(WordFactory.nullPointer()); - } + if (!SamplerSampleWriterDataAccess.initialize(data, skipCount, true)) { + return 0L; } - return 0L; + + assert SamplerSampleWriterDataAccess.verify(data); + assert data.getCurrentPos().unsignedRemainder(Long.BYTES).equal(0); + + /* + * Start a stack trace and do a stack walk. Note that the data will only be committed to + * the buffer if it is a new stack trace. + */ + SamplerSampleWriter.begin(data); + Pointer sp = KnownIntrinsics.readCallerStackPointer(); + CodePointer ip = FrameAccess.singleton().readReturnAddress(CurrentIsolate.getCurrentThread(), sp); + int errorCode = JfrStackWalker.walkCurrentThread(data, ip, sp, false); + return switch (errorCode) { + case JfrStackWalker.NO_ERROR, JfrStackWalker.TRUNCATED -> storeDeduplicatedStackTrace(data); + case JfrStackWalker.BUFFER_SIZE_EXCEEDED -> 0L; + case JfrStackWalker.UNPARSEABLE_STACK -> throw VMError.shouldNotReachHere("Only the async sampler may encounter an unparseable stack."); + default -> throw VMError.shouldNotReachHere("Unexpected return value"); + }; } finally { JfrExecutionSampler.singleton().allowSamplingInCurrentThread(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java new file mode 100644 index 000000000000..ca62dfbcbefb --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java @@ -0,0 +1,385 @@ +/* + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.jfr; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.FrameAccess; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.code.CodeInfoQueryResult; +import com.oracle.svm.core.code.CodeInfoTable; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.deopt.Deoptimizer; +import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.sampler.SamplerSampleWriter; +import com.oracle.svm.core.sampler.SamplerSampleWriterData; +import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; +import com.oracle.svm.core.stack.JavaFrame; +import com.oracle.svm.core.stack.JavaFrameAnchor; +import com.oracle.svm.core.stack.JavaFrameAnchors; +import com.oracle.svm.core.stack.JavaFrames; +import com.oracle.svm.core.stack.JavaStackWalk; +import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.PointerUtils; +import com.oracle.svm.core.util.VMError; + +/** + * Does a stack walk and records the instruction pointers of the physical Java frames that it + * encounters. Note that this class knows a lot of details about stack walking, so it needs to be in + * sync with {@link JavaStackWalker}. + * + * The code parts that are used for the async sampler need to be implemented in a very defensive way + * as the async sampler may encounter unexpected stack states. For this reason, this class may only + * use the unsafe methods of {@link FrameAccess} to read the return address. Otherwise, the + * validation in {@link FrameAccess} could fail. + */ +public final class JfrStackWalker { + /** A stack trace was recorded. */ + public static final int NO_ERROR = 0; + /** A stack trace was recorded, but it was truncated because we encountered too many frames. */ + public static final int TRUNCATED = 1; + /** No stack trace was recorded because the stack was unparseable. */ + public static final int UNPARSEABLE_STACK = 2; + /** No stack trace was recorded because it did not fit into the buffer. */ + public static final int BUFFER_SIZE_EXCEEDED = 3; + + @Platforms(Platform.HOSTED_ONLY.class) + private JfrStackWalker() { + } + + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + public static void walkCurrentThread(CodePointer initialIP, Pointer initialSP, boolean isAsync) { + SamplerSampleWriterData data = UnsafeStackValue.get(SamplerSampleWriterData.class); + if (SamplerSampleWriterDataAccess.initialize(data, 0, false)) { + SamplerSampleWriter.begin(data); + int result = walkCurrentThread(data, initialIP, initialSP, isAsync); + + switch (result) { + case NO_ERROR, TRUNCATED -> SamplerSampleWriter.end(data, SamplerSampleWriter.EXECUTION_SAMPLE_END); + case UNPARSEABLE_STACK -> { + VMError.guarantee(isAsync, "Only the async sampler may encounter an unparseable stack."); + JfrThreadLocal.increaseUnparseableStacks(); + } + case BUFFER_SIZE_EXCEEDED -> JfrThreadLocal.increaseMissedSamples(); + default -> throw VMError.shouldNotReachHere("Unexpected return value"); + } + } + } + + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + public static int walkCurrentThread(SamplerSampleWriterData data, CodePointer topFrameIP, Pointer topFrameSP, boolean isAsync) { + CodePointer ip = topFrameIP; + Pointer sp = topFrameSP; + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(); + + if (isAsync) { + if (VMThreads.SafepointBehavior.isCrashedThread(CurrentIsolate.getCurrentThread())) { + return UNPARSEABLE_STACK; + } + + CodeInfo codeInfo = CodeInfoTable.lookupImageCodeInfo(ip); + if (codeInfo.isNonNull()) { + /* + * We are in Java code, so the IP is accurate, and we can record it. However, it is + * possible that the IP is for a method that is usually not visible in a stack walk + * (e.g., if uninterruptible AOT-compiled code is somehow called + * via @CFunction(NO_TRANSITION)). In such a case, it is possible that we record an + * invalid/impossible stack trace because we use the frame anchors to continue the + * stack walk once we reach an entry point frame. This may result in skipped frames + * because there is no frame anchor for @CFunction(NO_TRANSITION) call. + */ + int result = recordIp(data, ip); + if (result != NO_ERROR) { + return result; + } + + /* + * We might be in the middle of pushing a new frame anchor. In that case, the top + * frame anchor will have invalid values and needs to be filtered out. + */ + anchor = filterTopFrameAnchorIfInvalid(anchor, topFrameSP); + + /* Move SP to the top of the caller frame. */ + long topFrameEncodedSize = CodeInfoAccess.lookupEncodedFrameSize(codeInfo, ip); + boolean topFrameIsEntryPoint = CodeInfoQueryResult.isEntryPoint(topFrameEncodedSize); + if (isSPAligned(sp)) { + UnsignedWord topFrameSize = WordFactory.unsigned(CodeInfoQueryResult.getTotalFrameSize(topFrameEncodedSize)); + if (SubstrateOptions.hasFramePointer() && !hasValidCaller(sp, topFrameSize, topFrameIsEntryPoint, anchor)) { + /* + * If we have a frame pointer, then the stack pointer can be aligned while + * we are in the method prologue/epilogue (i.e., the frame pointer and the + * return address are on top of the stack, but the actual stack frame is + * missing). We should reach the caller if we skip the incomplete top frame + * (frame pointer and return address). + */ + sp = sp.add(FrameAccess.wordSize() * 2); + } else { + /* + * Stack looks walkable - skip the top frame as we already recorded the + * corresponding IP. + */ + sp = sp.add(topFrameSize); + } + } else { + /* + * The async sampler interrupted the thread while it was in the middle of + * manipulating the stack (e.g., in a method prologue/epilogue). Most likely, + * there is a valid return address at the top of the stack that we can just + * skip. + */ + sp = sp.add(FrameAccess.wordSize()); + } + + /* Do a basic sanity check and decide if it makes sense to continue. */ + assert isSPAligned(sp); + if (!isCallerSPValid(topFrameSP, sp)) { + /* One of the assumptions above was incorrect. */ + return UNPARSEABLE_STACK; + } + + if (topFrameIsEntryPoint) { + /* + * If the top frame is for an entry point, then the caller needs to be treated + * like a native frame so that we are consistent with the normal stack walking + * code (i.e., we need to use the frame anchors). Note that the return address + * might point to AOT-compiled code though (e.g., if we did a @CFunction call to + * AOT-compiled code). + */ + if (anchor.isNull()) { + return UNPARSEABLE_STACK; + } + + sp = anchor.getLastJavaSP(); + ip = anchor.getLastJavaIP(); + anchor = anchor.getPreviousAnchor(); + } else { + /* + * The top frame and its caller are probably Java frames. Read the return + * address. + */ + ip = FrameAccess.singleton().unsafeReadReturnAddress(sp); + } + } else { + /* + * We are in native code, so we need to use the frame anchors to figure out where to + * start the stack walk. + */ + if (anchor.isNull()) { + /* + * The anchor is still null if the function is interrupted during the prologue + * (see: com.oracle.svm.core.graal.snippets.CFunctionSnippets.prologueSnippet) + * or if the thread called a native method without transition and without + * previous anchors. + */ + return UNPARSEABLE_STACK; + } + + /* + * Use the values from the frame anchor. If we are in native code that was called + * without a transition, we accept that we accidentally use the frame anchor of an + * older frame, which will result in a stack trace that misses the top frames. + */ + ip = anchor.getLastJavaIP(); + sp = anchor.getLastJavaSP(); + anchor = anchor.getPreviousAnchor(); + } + + /* + * Check if the SP and IP are sane enough to start a stack walk. For the async sampler, + * it is always possible to encounter a completely unexpected stack state. + */ + if (!isCallerValid(topFrameSP, sp, ip)) { + return UNPARSEABLE_STACK; + } + } + + return walkCurrentThread0(data, sp, ip, anchor, isAsync); + } + + /** + * When this method is called, we know that SP points into the stack of the current thread and + * IP points into AOT compiled code. + * + * If the async sampler is used, both values can still be incorrect though (i.e., they might + * just be sane enough so that we did not detect any obvious issues). Therefore, it can happen + * that we encounter invalid stack frames in the middle of the stack walk. We abort the stack + * walk in such a case. Note that it is possible that the stack walk finishes successfully even + * though it started with invalid data. We accept that we record an invalid/impossible stack + * trace in that case. + */ + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + private static int walkCurrentThread0(SamplerSampleWriterData data, Pointer startSP, CodePointer startIP, JavaFrameAnchor anchor, boolean isAsync) { + IsolateThread thread = CurrentIsolate.getCurrentThread(); + JavaStackWalk walk = UnsafeStackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); + JavaStackWalker.initialize(walk, thread, startSP, startIP, anchor); + assert JavaStackWalker.getEndSP(walk).isNull() : "not supported by the code below"; + + while (JavaStackWalker.advance(walk, thread)) { + JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); + VMError.guarantee(Deoptimizer.checkDeoptimized(frame) == null, "JIT compilation is not supported"); + + if (JavaFrames.isUnknownFrame(frame) || isAsync && !hasValidCaller(walk, frame)) { + /* Most likely, the stack walk already started with a wrong SP or IP. */ + return UNPARSEABLE_STACK; + } + + int result = recordIp(data, frame.getIP()); + if (result != NO_ERROR) { + return result; + } + } + + return NO_ERROR; + } + + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + private static int recordIp(SamplerSampleWriterData data, CodePointer ip) { + assert data.isNonNull(); + + /* Increment the number of seen frames. */ + data.setSeenFrames(data.getSeenFrames() + 1); + + if (shouldSkipFrame(data)) { + return NO_ERROR; + } else if (shouldTruncate(data)) { + return TRUNCATED; + } + + boolean success = SamplerSampleWriter.putLong(data, ip.rawValue()); + if (success) { + int newHash = computeHash(data.getHashCode(), ip.rawValue()); + data.setHashCode(newHash); + return NO_ERROR; + } + /* There was not enough space in the current buffer and no new/larger buffer. */ + return BUFFER_SIZE_EXCEEDED; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean shouldSkipFrame(SamplerSampleWriterData data) { + return data.getSeenFrames() <= data.getSkipCount(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean shouldTruncate(SamplerSampleWriterData data) { + int numFrames = data.getSeenFrames() - data.getSkipCount(); + if (numFrames > data.getMaxDepth()) { + /* The stack size exceeds given depth. Stop walk! */ + data.setTruncated(true); + return true; + } + return false; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static int computeHash(int oldHash, long ip) { + int hash = (int) (ip ^ (ip >>> 32)); + return 31 * oldHash + hash; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static JavaFrameAnchor filterTopFrameAnchorIfInvalid(JavaFrameAnchor anchor, Pointer currentSP) { + if (anchor.isNonNull() && !isCallerValid(currentSP, anchor.getLastJavaSP(), anchor.getLastJavaIP())) { + /* We are probably in the middle of pushing a frame anchor, so filter the top anchor. */ + return anchor.getPreviousAnchor(); + } + return anchor; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean hasValidCaller(JavaStackWalk walk, JavaFrame frame) { + return hasValidCaller(frame.getSP(), JavaFrames.getTotalFrameSize(frame), JavaFrames.isEntryPoint(frame), JavaStackWalker.getFrameAnchor(walk)); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean hasValidCaller(Pointer currentSP, UnsignedWord currentFrameSize, boolean currentFrameIsEntryPoint, JavaFrameAnchor anchor) { + if (currentFrameIsEntryPoint) { + /* + * The caller frame should belong to native code. However, we can't validate that the + * return address points into native code because AOT-compiled code may be called via + * a @CFunction call as well. So, we only do a basic sanity check of the frame anchor. + */ + return anchor.isNull() || anchor.getLastJavaSP().aboveThan(currentSP) && CodeInfoTable.isInAOTImageCode(anchor.getLastJavaIP()); + } else { + /* The caller frame should belong to Java code. */ + Pointer callerSP = currentSP.add(currentFrameSize); + if (!isCallerSPValid(currentSP, callerSP)) { + return false; + } + + /* Check if the return address points into AOT-compiled Java code. */ + CodePointer ip = FrameAccess.singleton().unsafeReadReturnAddress(callerSP); + return CodeInfoTable.isInAOTImageCode(ip); + } + } + + /** + * Check whether the given caller stack pointer (and the corresponding return address) are + * within the currently used part of the current thread's stack. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean isCallerSPValid(Pointer currentSP, Pointer callerSP) { + UnsignedWord stackEnd = VMThreads.StackEnd.get(); + UnsignedWord stackBase = VMThreads.StackBase.get(); + if (stackEnd.equal(0) || stackBase.equal(0)) { + /* Stack boundaries are unknown. */ + return false; + } + + assert stackEnd.belowThan(stackBase); + assert stackEnd.belowOrEqual(currentSP); + assert stackBase.aboveThan(currentSP); + + if (isSPAligned(callerSP) && callerSP.aboveThan(currentSP) && callerSP.belowThan(stackBase)) { + UnsignedWord returnAddressLocation = FrameAccess.singleton().unsafeReturnAddressLocation(callerSP); + return returnAddressLocation.aboveOrEqual(currentSP) && returnAddressLocation.belowThan(stackBase); + } + return false; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean isSPAligned(Pointer sp) { + return PointerUtils.isAMultiple(sp, WordFactory.unsigned(ConfigurationValues.getTarget().stackAlignment)); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean isCallerValid(Pointer currentSP, Pointer callerSP, CodePointer callerIP) { + return CodeInfoTable.isInAOTImageCode(callerIP) && isCallerSPValid(currentSP, callerSP); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java index 16f270e37af3..abafc3f6fa7a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrThreadLocal.java @@ -42,7 +42,6 @@ import com.oracle.svm.core.jfr.events.ThreadEndEvent; import com.oracle.svm.core.jfr.events.ThreadStartEvent; import com.oracle.svm.core.sampler.SamplerBuffer; -import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerStatistics; import com.oracle.svm.core.thread.JavaThreads; import com.oracle.svm.core.thread.PlatformThreads; @@ -94,12 +93,11 @@ public class JfrThreadLocal implements ThreadListener { private static final FastThreadLocalWord samplerBuffer = FastThreadLocalFactory.createWord("JfrThreadLocal.samplerBuffer"); private static final FastThreadLocalLong missedSamples = FastThreadLocalFactory.createLong("JfrThreadLocal.missedSamples"); private static final FastThreadLocalLong unparseableStacks = FastThreadLocalFactory.createLong("JfrThreadLocal.unparseableStacks"); - private static final FastThreadLocalWord samplerWriterData = FastThreadLocalFactory.createWord("JfrThreadLocal.samplerWriterData"); /* Non-thread-local fields. */ private static final JfrBufferList javaBufferList = new JfrBufferList(); private static final JfrBufferList nativeBufferList = new JfrBufferList(); - private long threadLocalBufferSize; + private UnsignedWord threadLocalBufferSize; @Fold public static JfrBufferList getNativeBufferList() { @@ -115,12 +113,12 @@ public static JfrBufferList getJavaBufferList() { public JfrThreadLocal() { } - public void initialize(long bufferSize) { + public void initialize(UnsignedWord bufferSize) { this.threadLocalBufferSize = bufferSize; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public long getThreadLocalBufferSize() { + public UnsignedWord getThreadLocalBufferSize() { return threadLocalBufferSize; } @@ -179,7 +177,6 @@ public static void stopRecording(IsolateThread isolateThread, boolean freeJavaBu missedSamples.set(isolateThread, 0); SamplerStatistics.singleton().addUnparseableSamples(getUnparseableStacks(isolateThread)); unparseableStacks.set(isolateThread, 0); - assert samplerWriterData.get(isolateThread).isNull(); SamplerBuffer buffer = samplerBuffer.get(isolateThread); if (buffer.isNonNull()) { @@ -332,7 +329,7 @@ public JfrBuffer getExistingJavaBuffer() { public JfrBuffer getJavaBuffer() { JfrBuffer buffer = javaBuffer.get(); if (buffer.isNull()) { - buffer = JfrBufferAccess.allocate(WordFactory.unsigned(threadLocalBufferSize), JfrBufferType.THREAD_LOCAL_JAVA); + buffer = JfrBufferAccess.allocate(threadLocalBufferSize, JfrBufferType.THREAD_LOCAL_JAVA); if (buffer.isNull()) { return WordFactory.nullPointer(); } @@ -351,7 +348,7 @@ public JfrBuffer getJavaBuffer() { public JfrBuffer getNativeBuffer() { JfrBuffer buffer = nativeBuffer.get(); if (buffer.isNull()) { - buffer = JfrBufferAccess.allocate(WordFactory.unsigned(threadLocalBufferSize), JfrBufferType.THREAD_LOCAL_NATIVE); + buffer = JfrBufferAccess.allocate(threadLocalBufferSize, JfrBufferType.THREAD_LOCAL_NATIVE); if (buffer.isNull()) { return WordFactory.nullPointer(); } @@ -499,15 +496,4 @@ public static long getUnparseableStacks() { public static long getUnparseableStacks(IsolateThread thread) { return unparseableStacks.get(thread); } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void setSamplerWriterData(SamplerSampleWriterData data) { - assert samplerWriterData.get().isNull() || data.isNull(); - samplerWriterData.set(data); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static SamplerSampleWriterData getSamplerWriterData() { - return samplerWriterData.get(); - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java index e49a55071735..eff3527c0480 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java @@ -235,8 +235,15 @@ public boolean createJFR(boolean simulateFailure) { options.validateAndAdjustMemoryOptions(); JfrTicks.initialize(); - threadLocal.initialize(options.threadBufferSize.getValue()); - globalMemory.initialize(options.globalBufferSize.getValue(), options.globalBufferCount.getValue()); + + long threadLocalBufferSize = options.threadBufferSize.getValue(); + assert threadLocalBufferSize > 0; + threadLocal.initialize(WordFactory.unsigned(threadLocalBufferSize)); + + long globalBufferSize = options.globalBufferSize.getValue(); + assert globalBufferSize > 0; + globalMemory.initialize(WordFactory.unsigned(globalBufferSize), options.globalBufferCount.getValue()); + unlockedChunkWriter.initialize(options.maxChunkSize.getValue()); stackTraceRepo.setStackTraceDepth(NumUtil.safeToInt(options.stackDepth.getValue())); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/AbstractJfrExecutionSampler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/AbstractJfrExecutionSampler.java index 67e8522d17e2..e57cfc50dd5a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/AbstractJfrExecutionSampler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/AbstractJfrExecutionSampler.java @@ -24,47 +24,27 @@ */ package com.oracle.svm.core.jfr.sampler; -import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; - import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; -import com.oracle.svm.core.FrameAccess; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.code.CodeInfo; -import com.oracle.svm.core.code.CodeInfoAccess; -import com.oracle.svm.core.code.CodeInfoQueryResult; -import com.oracle.svm.core.code.CodeInfoTable; -import com.oracle.svm.core.code.SimpleCodeInfoQueryResult; -import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jfr.JfrEvent; +import com.oracle.svm.core.jfr.JfrStackWalker; import com.oracle.svm.core.jfr.JfrThreadLocal; import com.oracle.svm.core.jfr.SubstrateJVM; -import com.oracle.svm.core.sampler.SamplerSampleWriter; -import com.oracle.svm.core.sampler.SamplerSampleWriterData; -import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; -import com.oracle.svm.core.sampler.SamplerStackWalkVisitor; -import com.oracle.svm.core.stack.JavaFrameAnchor; -import com.oracle.svm.core.stack.JavaFrameAnchors; -import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.thread.JavaVMOperation; import com.oracle.svm.core.thread.ThreadListener; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalInt; -import com.oracle.svm.core.util.PointerUtils; import jdk.graal.compiler.api.replacements.Fold; @@ -193,7 +173,7 @@ protected static boolean isExecutionSamplingAllowedInCurrentThread() { protected abstract void uninstall(IsolateThread thread); @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) - protected static void tryUninterruptibleStackWalk(CodePointer ip, Pointer sp) { + protected static void tryUninterruptibleStackWalk(CodePointer ip, Pointer sp, boolean isAsync) { /* * To prevent races, it is crucial that the thread count is incremented before we do any * other checks. @@ -204,7 +184,7 @@ protected static void tryUninterruptibleStackWalk(CodePointer ip, Pointer sp) { /* Prevent recursive sampler invocations during the stack walk. */ JfrExecutionSampler.singleton().preventSamplingInCurrentThread(); try { - doUninterruptibleStackWalk(ip, sp); + JfrStackWalker.walkCurrentThread(ip, sp, isAsync); } finally { JfrExecutionSampler.singleton().allowSamplingInCurrentThread(); } @@ -216,152 +196,6 @@ protected static void tryUninterruptibleStackWalk(CodePointer ip, Pointer sp) { } } - @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) - private static void doUninterruptibleStackWalk(CodePointer initialIp, Pointer initialSp) { - CodePointer ip = initialIp; - Pointer sp = initialSp; - if (!CodeInfoTable.isInAOTImageCode(ip)) { - JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(); - if (anchor.isNull()) { - /* - * The anchor is still null if the function is interrupted during prologue (see: - * com.oracle.svm.core.graal.snippets.CFunctionSnippets.prologueSnippet) or if java - * calls a native method without transition and without previous anchors. - */ - return; - } - - ip = anchor.getLastJavaIP(); - sp = anchor.getLastJavaSP(); - if (ip.isNull() || sp.isNull()) { - /* - * It can happen that anchor is in the list of all anchors, but its IP and SP are - * not filled yet. - */ - return; - } - } - - /* Try to do a stack walk. */ - SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class); - if (SamplerSampleWriterDataAccess.initialize(data, 0, false)) { - JfrThreadLocal.setSamplerWriterData(data); - try { - doUninterruptibleStackWalk(data, sp, ip); - } finally { - JfrThreadLocal.setSamplerWriterData(WordFactory.nullPointer()); - } - } - } - - /** - * Verify whether the stack pointer (SP) lies within the limits of the thread's stack. If not, - * attempting a stack walk might result in a segmentation fault (SEGFAULT). The stack pointer - * might be positioned outside the stack's boundaries if a signal interrupted the execution at - * the beginning of a method, before the SP was adjusted to its correct value. - */ - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static boolean isSPOutsideStackBoundaries(Pointer sp) { - UnsignedWord stackBase = VMThreads.StackBase.get(); - assert stackBase.notEqual(0); - Pointer returnAddressLocation = FrameAccess.singleton().getReturnAddressLocation(sp).add(FrameAccess.returnAddressSize()); - return returnAddressLocation.aboveThan(stackBase) || returnAddressLocation.belowOrEqual(VMThreads.StackEnd.get()); - } - - @Uninterruptible(reason = "This method must be uninterruptible since it uses untethered code info.", callerMustBe = true) - private static Pointer getCallerSP(CodeInfo codeInfo, Pointer sp, CodePointer ip) { - long relativeIP = CodeInfoAccess.relativeIP(codeInfo, ip); - long totalFrameSize = CodeInfoAccess.lookupTotalFrameSize(codeInfo, relativeIP); - return sp.add(WordFactory.unsigned(totalFrameSize)); - } - - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static boolean isSPAligned(Pointer sp) { - return PointerUtils.isAMultiple(sp, WordFactory.unsigned(ConfigurationValues.getTarget().stackAlignment)); - } - - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static boolean isEntryPoint(CodeInfo codeInfo, CodePointer ip) { - long relativeIP = CodeInfoAccess.relativeIP(codeInfo, ip); - SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class); - CodeInfoAccess.lookupCodeInfo(codeInfo, relativeIP, queryResult); - return CodeInfoQueryResult.isEntryPoint(queryResult.getEncodedFrameSize()); - } - - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static JavaFrameAnchor findLastJavaFrameAnchor(Pointer callerSP) { - JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(); - while (anchor.isNonNull() && anchor.getLastJavaSP().belowOrEqual(callerSP)) { - /* Skip anchors that are in parts of the stack we are not traversing. */ - anchor = anchor.getPreviousAnchor(); - } - assert anchor.isNull() || anchor.getLastJavaSP().aboveThan(callerSP); - return anchor; - } - - @Uninterruptible(reason = "This method must be uninterruptible since it uses untethered code info.", callerMustBe = true) - private static void doUninterruptibleStackWalk(SamplerSampleWriterData data, Pointer sp, CodePointer ip) { - SamplerSampleWriter.begin(data); - /* - * Visit the top frame. - * - * No matter where in the AOT-compiled code the signal has interrupted the execution, we - * know how to decode it. - */ - CodeInfo codeInfo = CodeInfoTable.getImageCodeInfo(ip); - SamplerStackWalkVisitor visitor = ImageSingletons.lookup(SamplerStackWalkVisitor.class); - if (!visitor.visitFrame(sp, ip, codeInfo, null, null)) { - /* The top frame is also the last one. */ - SamplerSampleWriter.end(data, SamplerSampleWriter.EXECUTION_SAMPLE_END); - return; - } - - Pointer callerSP; - if (isSPAligned(sp)) { - /* Stack is probably in a normal, walkable state. */ - callerSP = getCallerSP(codeInfo, sp, ip); - if (SubstrateOptions.hasFramePointer() && (isSPOutsideStackBoundaries(callerSP) || !CodeInfoTable.isInAOTImageCode(FrameAccess.singleton().readReturnAddress(callerSP)))) { - /* - * We are in the prologue or epilogue. Frame pointer and return address are on top - * of the stack. - */ - callerSP = sp.add(FrameAccess.wordSize()).add(FrameAccess.wordSize()); - } - } else { - /* We are in the prologue or epilogue. Return address is at the top of the stack. */ - callerSP = sp.add(FrameAccess.wordSize()); - } - - if (isSPOutsideStackBoundaries(callerSP)) { - /* We made an incorrect assumption earlier, the stack is not walkable. */ - JfrThreadLocal.increaseUnparseableStacks(); - return; - } - - CodePointer returnAddressIP = FrameAccess.singleton().readReturnAddress(callerSP); - if (!CodeInfoTable.isInAOTImageCode(returnAddressIP)) { - if (isEntryPoint(codeInfo, ip)) { - JavaFrameAnchor anchor = findLastJavaFrameAnchor(callerSP); - if (anchor.isNonNull()) { - callerSP = anchor.getLastJavaSP(); - returnAddressIP = anchor.getLastJavaIP(); - } else { - SamplerSampleWriter.end(data, SamplerSampleWriter.EXECUTION_SAMPLE_END); - return; - } - } else { - /* We made an incorrect assumption earlier, the stack is not walkable. */ - JfrThreadLocal.increaseUnparseableStacks(); - return; - } - } - - /* Start a stack walk but from the frame after the top one. */ - if (JavaStackWalker.walkCurrentThread(callerSP, returnAddressIP, visitor) || data.getTruncated()) { - SamplerSampleWriter.end(data, SamplerSampleWriter.EXECUTION_SAMPLE_END); - } - } - /** * Starts/Stops execution sampling and updates the sampling interval. * diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java index edf7d1ce6f99..deba2e7e368b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java @@ -141,7 +141,7 @@ private static final class ExecutionSampleCallback implements Threading.Recurrin public void run(Threading.RecurringCallbackAccess access) { Pointer sp = readCallerStackPointer(); CodePointer ip = readReturnAddress(); - tryUninterruptibleStackWalk(ip, sp); + tryUninterruptibleStackWalk(ip, sp, false); } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java index 0423aa76e20d..649fb633a48f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java @@ -25,6 +25,8 @@ package com.oracle.svm.core.sampler; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -45,10 +47,15 @@ private SamplerBufferAccess() { } @Fold - public static UnsignedWord getHeaderSize() { + static UnsignedWord getHeaderSize() { return UnsignedUtils.roundUp(SizeOf.unsigned(SamplerBuffer.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getTotalBufferSize(UnsignedWord dataSize) { + return SamplerBufferAccess.getHeaderSize().add(dataSize); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void reinitialize(SamplerBuffer buffer) { assert buffer.isNonNull(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java index 460fdecf95ba..53e58add5ed7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -158,10 +158,8 @@ private boolean allocateAndPush() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private SamplerBuffer tryAllocateBuffer0() { - UnsignedWord headerSize = SamplerBufferAccess.getHeaderSize(); - UnsignedWord dataSize = WordFactory.unsigned(SubstrateJVM.getThreadLocal().getThreadLocalBufferSize()); - - SamplerBuffer result = NullableNativeMemory.malloc(headerSize.add(dataSize), NmtCategory.JFR); + UnsignedWord dataSize = SubstrateJVM.getThreadLocal().getThreadLocalBufferSize(); + SamplerBuffer result = NullableNativeMemory.malloc(SamplerBufferAccess.getTotalBufferSize(dataSize), NmtCategory.JFR); if (result.isNonNull()) { bufferCount++; result.setSize(dataSize); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java index 5c072824251d..0a05f039c21a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java @@ -122,16 +122,16 @@ public interface SamplerSampleWriterData extends PointerBase { void setMaxDepth(int value); /** - * Returns the number of frames. + * Returns the number of frames that were visited or skipped. */ @RawField - int getNumFrames(); + int getSeenFrames(); /** - * Sets the number of frames. + * Sets the number of frames that were visited or skipped. */ @RawField - void setNumFrames(int value); + void setSeenFrames(int value); /** * Returns {@code true} if the stack size exceeds {@link #getMaxDepth()}. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterDataAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterDataAccess.java index 4472c10d1dca..54aba8121936 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterDataAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterDataAccess.java @@ -63,7 +63,7 @@ public static boolean initialize(SamplerSampleWriterData data, int skipCount, bo * Initialize the {@link SamplerSampleWriterData data} so that it uses the given buffer. */ @Uninterruptible(reason = "Accesses a sampler buffer", callerMustBe = true) - private static void initialize0(SamplerSampleWriterData data, SamplerBuffer buffer, int skipCount, int maxDepth, boolean allowBufferAllocation) { + public static void initialize0(SamplerSampleWriterData data, SamplerBuffer buffer, int skipCount, int maxDepth, boolean allowBufferAllocation) { assert SamplerBufferAccess.verify(buffer); data.setSamplerBuffer(buffer); @@ -74,7 +74,7 @@ private static void initialize0(SamplerSampleWriterData data, SamplerBuffer buff data.setMaxDepth(maxDepth); data.setTruncated(false); data.setSkipCount(skipCount); - data.setNumFrames(0); + data.setSeenFrames(0); data.setAllowBufferAllocation(allowBufferAllocation); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java deleted file mode 100644 index b0d480b09def..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.sampler; - -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.word.Pointer; - -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.code.CodeInfo; -import com.oracle.svm.core.deopt.DeoptimizedFrame; -import com.oracle.svm.core.jfr.JfrThreadLocal; -import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; - -/* Uninterruptible visitor that holds all its state in a thread-local because it is used concurrently by multiple threads. */ -public final class SamplerStackWalkVisitor extends ParameterizedStackFrameVisitor { - @Platforms(Platform.HOSTED_ONLY.class) - public SamplerStackWalkVisitor() { - } - - @Override - @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) - public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Object data) { - return recordIp(ip); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean shouldContinueWalk(SamplerSampleWriterData data) { - int numFrames = data.getNumFrames() - data.getSkipCount(); - if (numFrames > data.getMaxDepth()) { - /* The stack size exceeds given depth. Stop walk! */ - data.setTruncated(true); - return false; - } else { - return true; - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean shouldSkipFrame(SamplerSampleWriterData data) { - data.setNumFrames(data.getNumFrames() + 1); - return data.getNumFrames() <= data.getSkipCount(); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static int computeHash(int oldHash, long ip) { - int hash = (int) (ip ^ (ip >>> 32)); - return 31 * oldHash + hash; - } - - @Override - @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) - protected boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Object data) { - /* - * The SIGPROF-based sampler may interrupt at any arbitrary code location. The stack - * information that we currently have is not always good enough to do a reliable stack walk. - */ - JfrThreadLocal.increaseUnparseableStacks(); - return false; - } - - @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) - private static boolean recordIp(CodePointer ip) { - SamplerSampleWriterData writerData = JfrThreadLocal.getSamplerWriterData(); - assert writerData.isNonNull(); - - boolean shouldSkipFrame = shouldSkipFrame(writerData); - boolean shouldContinueWalk = shouldContinueWalk(writerData); - if (!shouldSkipFrame && shouldContinueWalk) { - writerData.setHashCode(computeHash(writerData.getHashCode(), ip.rawValue())); - shouldContinueWalk = SamplerSampleWriter.putLong(writerData, ip.rawValue()); - } - return shouldContinueWalk; - } -} From dfc4db490068947ee60a85c004a914267e97184b Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 10 May 2024 12:34:07 +0200 Subject: [PATCH 6/7] Fixed a minor performance issue in the serial GC. Various fixes for @Uninterruptible. --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 2 +- .../svm/core/genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../src/com/oracle/svm/core/genscavenge/HeapChunk.java | 4 +++- .../com/oracle/svm/core/genscavenge/ImageHeapWalker.java | 9 +++++---- .../genscavenge/remset/AlignedChunkRememberedSet.java | 2 +- .../genscavenge/remset/UnalignedChunkRememberedSet.java | 2 +- .../svm/core/heap/InstanceReferenceMapDecoder.java | 1 + .../com/oracle/svm/core/heap/PodReferenceMapDecoder.java | 1 + .../oracle/svm/core/heap/StoredContinuationAccess.java | 7 ++++--- .../com/oracle/svm/core/hub/InteriorObjRefWalker.java | 9 ++++++--- 10 files changed, 24 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 70f0417167a8..d79e5cc736e4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -955,7 +955,7 @@ private void blackenImageHeapRoots() { } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { walkImageHeapRoots(imageHeapInfo, greyToBlackObjectVisitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index bc432b866261..02113f97c5f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -56,7 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 98157657db81..a608f1102a14 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -294,12 +294,13 @@ private static SignedWord offsetFromPointer(Header that, PointerBase pointer) } @NeverInline("Not performance critical") + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") public static boolean walkObjectsFrom(Header that, Pointer start, ObjectVisitor visitor) { return walkObjectsFromInline(that, start, visitor); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public static boolean walkObjectsFromInline(Header that, Pointer start, ObjectVisitor visitor) { Pointer p = start; while (p.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read @@ -312,6 +313,7 @@ public static boolean walkObjectsFromInline(Header that, Pointer start, Objec return true; } + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitor(ObjectVisitor visitor, Object obj) { return visitor.visitObjectInline(obj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index 03d4e143c304..38a1dcecb19c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -66,18 +66,19 @@ public static boolean walkImageHeapObjects(ImageHeapInfo heapInfo, ObjectVisitor walkPartition(heapInfo.firstReadOnlyHugeObject, heapInfo.lastReadOnlyHugeObject, visitor, false); } + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, false); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) { if (firstObject == null || lastObject == null) { assert firstObject == null && lastObject == null; @@ -129,6 +130,7 @@ private static boolean visitObject(ObjectVisitor visitor, Object currentObject) return visitor.visitObject(currentObject); } + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean visitObjectInline(ObjectVisitor visitor, Object currentObject) { return visitor.visitObjectInline(currentObject); @@ -178,10 +180,9 @@ public boolean consistsOfHugeObjects(ImageHeapInfo region) { } @Override - @AlwaysInline("GC performance") public final boolean visitObjects(ImageHeapInfo region, ObjectVisitor visitor) { boolean alignedChunks = !consistsOfHugeObjects; - return ImageHeapWalker.walkPartitionInline(getFirstObject(region), getLastObject(region), visitor, alignedChunks); + return ImageHeapWalker.walkPartition(getFirstObject(region), getLastObject(region), visitor, alignedChunks); } protected abstract Object getFirstObject(ImageHeapInfo info); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index 7efb7222bf00..9d40e551ca05 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -196,7 +196,7 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 5fb160529b47..b77848d57870 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -91,7 +91,7 @@ public static void dirtyCardForObject(Object obj, boolean verifyOnly) { } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") public static void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer rememberedSetStart = getCardTableStart(chunk); UnsignedWord objectIndex = getObjectIndex(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index d7a10f67ec75..55ebdabaa3e0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -73,6 +73,7 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra return true; } + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitor(ObjectReferenceVisitor visitor, Object holderObject, boolean compressed, Pointer objRef) { return visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index ef9854e70b12..7e2afe0f586e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -74,6 +74,7 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco return true; } + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitor(Pointer baseAddress, ObjectReferenceVisitor visitor, Object obj, boolean isCompressed, UnsignedWord refOffset) { return visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index fcfe1f74e2f5..656ac3bd0a63 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -170,7 +170,7 @@ private static void setIP(StoredContinuation cont, CodePointer ip) { } @AlwaysInline("De-virtualize calls to ObjectReferenceVisitor") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisitor visitor) { assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; @@ -197,7 +197,7 @@ public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisito } @AlwaysInline("De-virtualize calls to visitor.") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) public static void walkFrames(StoredContinuation s, ContinuationStackFrameVisitor visitor, ContinuationStackFrameVisitorData data) { assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; @@ -225,7 +225,7 @@ public static void walkFrames(StoredContinuation s, ContinuationStackFrameVisito } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) public static void walkFrameReferences(JavaFrame frame, CodeInfo codeInfo, ObjectReferenceVisitor visitor, Object holderObject) { NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); long referenceMapIndex = frame.getReferenceMapIndex(); @@ -235,6 +235,7 @@ public static void walkFrameReferences(JavaFrame frame, CodeInfo codeInfo, Objec } public abstract static class ContinuationStackFrameVisitor { + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) public abstract void visitFrame(ContinuationStackFrameVisitorData data, Pointer sp, NonmovableArray referenceMapEncoding, long referenceMapIndex, ContinuationStackFrameVisitor visitor); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index c0ef0a3b9f19..8f7d1383f333 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.heap.PodReferenceMapDecoder; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceInternals; +import com.oracle.svm.core.heap.StoredContinuation; import com.oracle.svm.core.heap.StoredContinuationAccess; import com.oracle.svm.core.thread.ContinuationSupport; import com.oracle.svm.core.util.VMError; @@ -64,12 +65,13 @@ public class InteriorObjRefWalker { * @return True if the walk was successful, or false otherwise. */ @NeverInline("Non-performance critical version") + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") public static boolean walkObject(final Object obj, final ObjectReferenceVisitor visitor) { return walkObjectInline(obj, visitor); } @AlwaysInline("Performance critical version") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -143,12 +145,12 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!ContinuationSupport.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); } - return StoredContinuationAccess.walkReferences(obj, visitor); + return StoredContinuationAccess.walkReferences((StoredContinuation) obj, visitor); } @AlwaysInline("Performance critical version") @@ -176,6 +178,7 @@ private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visito return true; } + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitor(Object obj, ObjectReferenceVisitor visitor, boolean isCompressed, Pointer pos) { return visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); From ca7c882c6c75cc48547452966b736d89bd5ed7d6 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 29 May 2024 15:14:38 +0200 Subject: [PATCH 7/7] Cleanups. --- .../genscavenge/CompactingOldGeneration.java | 5 +- .../oracle/svm/core/genscavenge/GCImpl.java | 7 +- .../genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../svm/core/genscavenge/HeapChunk.java | 4 +- .../svm/core/genscavenge/ImageHeapWalker.java | 6 +- .../remset/AlignedChunkRememberedSet.java | 2 +- .../remset/UnalignedChunkRememberedSet.java | 2 +- .../src/com/oracle/svm/core/FrameAccess.java | 6 +- .../oracle/svm/core/code/CodeInfoAccess.java | 13 +-- .../svm/core/code/CodeInfoQueryResult.java | 4 +- .../oracle/svm/core/code/CodeInfoTable.java | 1 + .../oracle/svm/core/deopt/Deoptimizer.java | 64 +++++++-------- .../svm/core/heap/StoredContinuation.java | 8 +- .../core/heap/StoredContinuationAccess.java | 8 +- .../svm/core/hub/InteriorObjRefWalker.java | 4 +- .../jdk/Target_java_lang_StackWalker.java | 21 +++-- .../oracle/svm/core/jfr/JfrStackWalker.java | 81 +++++++++---------- .../svm/core/snippets/ExceptionUnwind.java | 6 +- .../svm/core/stack/JavaFrameAnchors.java | 8 +- .../oracle/svm/core/stack/JavaStackWalk.java | 4 +- .../svm/core/stack/JavaStackWalker.java | 46 +++++------ .../stack/ParameterizedStackFrameVisitor.java | 6 +- .../svm/core/stack/ThreadStackPrinter.java | 2 + 23 files changed, 162 insertions(+), 148 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java index 80b43f4011bb..f0de34f37a7f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java @@ -25,12 +25,10 @@ package com.oracle.svm.core.genscavenge; import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; -import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -365,8 +363,7 @@ private void fixupUnalignedChunkReferences(ChunkReleaser chunkReleaser) { @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.") private void fixupStackReferences() { Pointer sp = readCallerStackPointer(); - CodePointer ip = readReturnAddress(); - GCImpl.walkStackRoots(refFixupVisitor, sp, ip, false); + GCImpl.walkStackRoots(refFixupVisitor, sp, false); } private void compact(Timers timers) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index d79e5cc736e4..654106ac6cd0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.genscavenge; import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.lang.ref.Reference; import org.graalvm.nativeimage.CurrentIsolate; @@ -799,7 +800,7 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) private void blackenStackRoots() { Timer blackenStackRootsTimer = timers.blackenStackRoots.open(); try { - Pointer sp = readCallerStackPointer(); + Pointer sp = KnownIntrinsics.readCallerStackPointer(); walkStackRoots(greyToBlackObjRefVisitor, sp, true); } finally { blackenStackRootsTimer.close(); @@ -814,7 +815,7 @@ static void walkStackRoots(ObjectReferenceVisitor visitor, Pointer currentThread * anchor). */ JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); - JavaStackWalker.initialize(walk, CurrentIsolate.getCurrentThread(), sp); + JavaStackWalker.initialize(walk, CurrentIsolate.getCurrentThread(), currentThreadSp); walkStack(CurrentIsolate.getCurrentThread(), walk, visitor, visitRuntimeCodeInfo); /* @@ -955,7 +956,7 @@ private void blackenImageHeapRoots() { } } - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { walkImageHeapRoots(imageHeapInfo, greyToBlackObjectVisitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 02113f97c5f5..386144ec3027 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -56,7 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index a608f1102a14..50a42e9a5f9c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -294,13 +294,13 @@ private static SignedWord offsetFromPointer(Header that, PointerBase pointer) } @NeverInline("Not performance critical") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") public static boolean walkObjectsFrom(Header that, Pointer start, ObjectVisitor visitor) { return walkObjectsFromInline(that, start, visitor); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public static boolean walkObjectsFromInline(Header that, Pointer start, ObjectVisitor visitor) { Pointer p = start; while (p.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index 38a1dcecb19c..da58392793c8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -66,19 +66,19 @@ public static boolean walkImageHeapObjects(ImageHeapInfo heapInfo, ObjectVisitor walkPartition(heapInfo.firstReadOnlyHugeObject, heapInfo.lastReadOnlyHugeObject, visitor, false); } - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, false); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) { if (firstObject == null || lastObject == null) { assert firstObject == null && lastObject == null; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index 9d40e551ca05..558e1d046741 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -196,7 +196,7 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito } } - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index b77848d57870..43adea759c2a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -91,7 +91,7 @@ public static void dirtyCardForObject(Object obj, boolean verifyOnly) { } } - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") public static void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer rememberedSetStart = getCardTableStart(chunk); UnsignedWord objectIndex = getObjectIndex(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java index f604f9c0e473..c6b86b8f9076 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java @@ -50,12 +50,12 @@ /** * This class can be used to access physical Java frames. It cannot be used to access virtual Java * frames, and it must not be used to access physical native frames (we can't assume a specific - * layout for native frames, especially on platform such as aarch64). + * layout for native frames, especially on platforms such as aarch64). *

* When accessing a return address, note that the return address belongs to a different frame than * the stack pointer (SP) because the return address is located at a negative offset relative to SP. - * For Java frames that called native code, the return address is located in the native frame and - * can't be accessed because we can't assume a specific layout for native frames. + * For Java frames that called native code, the return address is located in the native callee frame + * and can't be accessed because we can't assume a specific layout for native frames. */ public abstract class FrameAccess { @Fold diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java index 4e158b5ddf33..719b86f47663 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java @@ -268,16 +268,12 @@ public static long lookupDeoptimizationEntrypoint(CodeInfo info, long method, lo @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long lookupTotalFrameSize(CodeInfo info, long relativeIP) { - SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); - CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult); - return CodeInfoQueryResult.getTotalFrameSize(codeInfoQueryResult.getEncodedFrameSize()); + return CodeInfoQueryResult.getTotalFrameSize(lookupEncodedFrameSize(info, relativeIP)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long lookupTotalFrameSize(CodeInfo info, CodePointer ip) { - SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); - lookupCodeInfo(info, ip, codeInfoQueryResult); - return CodeInfoQueryResult.getTotalFrameSize(codeInfoQueryResult.getEncodedFrameSize()); + return CodeInfoQueryResult.getTotalFrameSize(lookupEncodedFrameSize(info, ip)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -312,6 +308,11 @@ public static void lookupCodeInfo(CodeInfo info, CodePointer ip, SimpleCodeInfoQ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long lookupEncodedFrameSize(CodeInfo info, CodePointer ip) { long relativeIP = CodeInfoAccess.relativeIP(info, ip); + return lookupEncodedFrameSize(info, relativeIP); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static long lookupEncodedFrameSize(CodeInfo info, long relativeIP) { SimpleCodeInfoQueryResult codeInfoQueryResult = UnsafeStackValue.get(SimpleCodeInfoQueryResult.class); CodeInfoDecoder.lookupCodeInfo(info, relativeIP, codeInfoQueryResult); return codeInfoQueryResult.getEncodedFrameSize(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java index 937867d9dc5b..0dcbbc897d95 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoQueryResult.java @@ -32,8 +32,8 @@ /** * Information about an instruction pointer (IP), created and returned by methods in - * {@link CodeInfoTable}. In situations, where we can't allocate any Java heap memory, we use the - * class {@link SimpleCodeInfoQueryResult} instead. + * {@link CodeInfoTable}. In situations where we can't allocate any Java heap memory, we use the + * structure {@link SimpleCodeInfoQueryResult} instead. * * During a stack walk, this class holds information about a physical Java frame (see * {@link FrameInfoQueryResult} for the virtual Java frames). diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index 868164369398..bdc1dfc5b1ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -137,6 +137,7 @@ public static CodeInfoQueryResult lookupDeoptimizationEntrypoint(CodeInfo info, return result; } + /** Note that this method is only called for regular frames but not for deoptimized frames. */ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo info, ObjectReferenceVisitor visitor) { counters().visitObjectReferencesCount.inc(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java index c11686150120..de864b9dfad0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java @@ -239,10 +239,10 @@ public static DeoptimizedFrame checkDeoptimized(JavaFrame frame) { } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static DeoptimizedFrame checkDeoptimized(IsolateThread thread, Pointer sourceSp) { + public static DeoptimizedFrame checkDeoptimized(IsolateThread thread, Pointer sp) { if (DeoptimizationSupport.enabled()) { - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(thread, sourceSp); - return checkDeoptimized0(sourceSp, returnAddress); + CodePointer ip = FrameAccess.singleton().readReturnAddress(thread, sp); + return checkDeoptimized0(sp, ip); } return null; } @@ -252,13 +252,13 @@ public static DeoptimizedFrame checkDeoptimized(IsolateThread thread, Pointer so * returns the {@link DeoptimizedFrame} in that case. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static DeoptimizedFrame checkDeoptimized0(Pointer sourceSp, CodePointer returnAddress) { + private static DeoptimizedFrame checkDeoptimized0(Pointer sp, CodePointer ip) { /* A frame is deoptimized when the return address was patched to the deoptStub. */ - if (returnAddress.equal(DeoptimizationSupport.getDeoptStubPointer())) { + if (ip.equal(DeoptimizationSupport.getDeoptStubPointer())) { /* The DeoptimizedFrame instance is stored above the return address. */ - DeoptimizedFrame result = (DeoptimizedFrame) ReferenceAccess.singleton().readObjectAt(sourceSp, true); + DeoptimizedFrame result = (DeoptimizedFrame) ReferenceAccess.singleton().readObjectAt(sp, true); if (result == null) { - throw checkDeoptimizedError(sourceSp); + throw checkDeoptimizedError(sp); } return result; } @@ -266,13 +266,13 @@ private static DeoptimizedFrame checkDeoptimized0(Pointer sourceSp, CodePointer } @Uninterruptible(reason = "Switch to interruptible code and report a fatal error.", calleeMustBe = false) - private static RuntimeException checkDeoptimizedError(Pointer sourceSp) { - throw checkDeoptimizedError0(sourceSp); + private static RuntimeException checkDeoptimizedError(Pointer sp) { + throw checkDeoptimizedError0(sp); } @NeverInline("Throws error and exits") - private static RuntimeException checkDeoptimizedError0(Pointer sourceSp) { - Log.log().string("Unable to retrieve Deoptimized frame. sp: ").hex(sourceSp.rawValue()).newline(); + private static RuntimeException checkDeoptimizedError0(Pointer sp) { + Log.log().string("Unable to retrieve Deoptimized frame. sp: ").hex(sp.rawValue()).newline(); throw VMError.shouldNotReachHere("Unable to retrieve Deoptimized frame"); } @@ -285,8 +285,8 @@ private void installDeoptimizedFrame(DeoptimizedFrame deoptimizedFrame) { /* * Store a pointer to the deoptimizedFrame in the stack slot above the return address. From - * this point on, the GC will ignore the original source frame content. Instead, it just - * visits the DeoptimizedFrame object. + * this point on, the GC will ignore the original source frame content. Instead, it will + * visit the DeoptimizedFrame object. */ ReferenceAccess.singleton().writeObjectAt(sourceSp, deoptimizedFrame, true); } @@ -377,20 +377,20 @@ protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStu * instead of raising an error (use for deoptimzation testing only). */ @NeverInline("Inlining of this method would require that we have deopt targets for callees of this method (SVM internals).") - public static void deoptimizeFrame(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation) { + public static void deoptimizeFrame(Pointer sp, boolean ignoreNonDeoptimizable, SpeculationReason speculation) { /* * Note that the thread needs to be read outside of the VMOperation, since the operation can * run in any different thread. */ IsolateThread targetThread = CurrentIsolate.getCurrentThread(); - DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(targetThread, sourceSp); + DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(targetThread, sp); if (deoptFrame != null) { /* Already deoptimized, so nothing to do. */ registerSpeculationFailure(deoptFrame.getSourceInstalledCode(), speculation); return; } - DeoptimizeFrameOperation vmOp = new DeoptimizeFrameOperation(sourceSp, ignoreNonDeoptimizable, speculation, targetThread); + DeoptimizeFrameOperation vmOp = new DeoptimizeFrameOperation(sp, ignoreNonDeoptimizable, speculation, targetThread); vmOp.enqueue(); } @@ -411,32 +411,32 @@ private static class DeoptimizeFrameOperation extends JavaVMOperation { @Override protected void operate() { VMOperation.guaranteeInProgress("doDeoptimizeFrame"); - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(targetThread, sourceSp); - deoptimizeFrame(sourceSp, ignoreNonDeoptimizable, speculation, returnAddress, targetThread); + CodePointer ip = FrameAccess.singleton().readReturnAddress(targetThread, sourceSp); + deoptimizeFrame(targetThread, sourceSp, ip, ignoreNonDeoptimizable, speculation); } } @Uninterruptible(reason = "Prevent the GC from freeing the CodeInfo object.") - private static void deoptimizeFrame(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodePointer returnAddress, IsolateThread targetThread) { - UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(returnAddress); + private static void deoptimizeFrame(IsolateThread targetThread, Pointer sp, CodePointer ip, boolean ignoreNonDeoptimizable, SpeculationReason speculation) { + UntetheredCodeInfo untetheredInfo = CodeInfoTable.lookupCodeInfo(ip); Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); - deoptimize(sourceSp, ignoreNonDeoptimizable, speculation, returnAddress, info, targetThread); + deoptimize(targetThread, sp, ip, ignoreNonDeoptimizable, speculation, info); } finally { CodeInfoAccess.releaseTether(untetheredInfo, tether); } } @Uninterruptible(reason = "Pass the now protected CodeInfo object to interruptible code.", calleeMustBe = false) - private static void deoptimize(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodePointer returnAddress, CodeInfo info, IsolateThread targetThread) { - deoptimize0(sourceSp, ignoreNonDeoptimizable, speculation, returnAddress, info, targetThread); + private static void deoptimize(IsolateThread targetThread, Pointer sp, CodePointer ip, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodeInfo info) { + deoptimize0(targetThread, sp, ip, ignoreNonDeoptimizable, speculation, info); } - private static void deoptimize0(Pointer sourceSp, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodePointer returnAddress, CodeInfo info, IsolateThread targetThread) { - CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(info, returnAddress); - Deoptimizer deoptimizer = new Deoptimizer(sourceSp, queryResult, targetThread); - DeoptimizedFrame sourceFrame = deoptimizer.deoptSourceFrame(returnAddress, ignoreNonDeoptimizable); + private static void deoptimize0(IsolateThread targetThread, Pointer sp, CodePointer ip, boolean ignoreNonDeoptimizable, SpeculationReason speculation, CodeInfo info) { + CodeInfoQueryResult queryResult = CodeInfoTable.lookupCodeInfoQueryResult(info, ip); + Deoptimizer deoptimizer = new Deoptimizer(sp, queryResult, targetThread); + DeoptimizedFrame sourceFrame = deoptimizer.deoptSourceFrame(ip, ignoreNonDeoptimizable); if (sourceFrame != null) { registerSpeculationFailure(sourceFrame.getSourceInstalledCode(), speculation); } @@ -446,9 +446,9 @@ private static void deoptimize0(Pointer sourceSp, boolean ignoreNonDeoptimizable * Invalidates the {@link InstalledCode} of the method of the given frame. The method must be a * runtime compiled method, since there is not {@link InstalledCode} for native image methods. */ - public static void invalidateMethodOfFrame(IsolateThread thread, Pointer sourceSp, SpeculationReason speculation) { - CodePointer returnAddress = FrameAccess.singleton().readReturnAddress(thread, sourceSp); - SubstrateInstalledCode installedCode = CodeInfoTable.lookupInstalledCode(returnAddress); + public static void invalidateMethodOfFrame(IsolateThread thread, Pointer sp, SpeculationReason speculation) { + CodePointer ip = FrameAccess.singleton().readReturnAddress(thread, sp); + SubstrateInstalledCode installedCode = CodeInfoTable.lookupInstalledCode(ip); /* * We look up the installedCode before checking if the frame is deoptimized to avoid race * conditions. We are not in a VMOperation here. When a deoptimization happens, e.g., at a @@ -458,7 +458,7 @@ public static void invalidateMethodOfFrame(IsolateThread thread, Pointer sourceS * installedCode multiple times in case of a race is not a problem because the actual * invalidation is in a VMOperation. */ - DeoptimizedFrame deoptimizedFrame = checkDeoptimized(thread, sourceSp); + DeoptimizedFrame deoptimizedFrame = checkDeoptimized(thread, sp); if (deoptimizedFrame != null) { installedCode = deoptimizedFrame.getSourceInstalledCode(); if (installedCode == null) { @@ -468,7 +468,7 @@ public static void invalidateMethodOfFrame(IsolateThread thread, Pointer sourceS } else { if (installedCode == null) { throw VMError.shouldNotReachHere( - "Only runtime compiled methods can be invalidated. sp = " + Long.toHexString(sourceSp.rawValue()) + ", returnAddress = " + Long.toHexString(returnAddress.rawValue())); + "Only runtime compiled methods can be invalidated. sp = " + Long.toHexString(sp.rawValue()) + ", returnAddress = " + Long.toHexString(ip.rawValue())); } } registerSpeculationFailure(installedCode, speculation); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java index 0085793591d0..bb4b36a7ecda 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java @@ -24,11 +24,15 @@ */ package com.oracle.svm.core.heap; +import org.graalvm.nativeimage.c.function.CodePointer; + import com.oracle.svm.core.hub.Hybrid; + import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.c.function.CodePointer; -/** Execution state of a continuation, use via {@link StoredContinuationAccess}. */ +/** + * Persisted execution state of a yielded continuation, use via {@link StoredContinuationAccess}. + */ @Hybrid(componentType = Word.class) public final class StoredContinuation { CodePointer ip; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index 656ac3bd0a63..8e1505c2afa6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -175,9 +175,9 @@ public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisito assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); - JavaStackWalker.initialize(walk, s); + JavaStackWalker.initializeForContinuation(walk, s); - while (JavaStackWalker.advance(walk, s)) { + while (JavaStackWalker.advanceForContinuation(walk, s)) { JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); VMError.guarantee(!JavaFrames.isEntryPoint(frame), "Entry point frames are not supported"); VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); @@ -202,9 +202,9 @@ public static void walkFrames(StoredContinuation s, ContinuationStackFrameVisito assert !Heap.getHeap().isInImageHeap(s) : "StoredContinuations in the image heap are read-only and don't need to be visited"; JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); - JavaStackWalker.initialize(walk, s); + JavaStackWalker.initializeForContinuation(walk, s); - while (JavaStackWalker.advance(walk, s)) { + while (JavaStackWalker.advanceForContinuation(walk, s)) { JavaFrame frame = JavaStackWalker.getCurrentFrame(walk); VMError.guarantee(!JavaFrames.isEntryPoint(frame), "Entry point frames are not supported"); VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "Stack walk must not encounter unknown frame"); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 8f7d1383f333..c86044d5a763 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -65,13 +65,13 @@ public class InteriorObjRefWalker { * @return True if the walk was successful, or false otherwise. */ @NeverInline("Non-performance critical version") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).") + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") public static boolean walkObject(final Object obj, final ObjectReferenceVisitor visitor) { return walkObjectInline(obj, visitor); } @AlwaysInline("Performance critical version") - @Uninterruptible(reason = "Due to forced inlining (StoredContinuation objects must not move).", callerMustBe = true) + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java index e23f2686b8f5..403c1dfef17d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_StackWalker.java @@ -245,6 +245,7 @@ final class ContinuationSpliterator extends AbstractStackFrameSpliterator { private boolean initialized; private JavaStackWalk walk; private Target_jdk_internal_vm_Continuation continuation; + private StoredContinuation stored; ContinuationSpliterator(JavaStackWalk walk, Target_jdk_internal_vm_ContinuationScope contScope, Target_jdk_internal_vm_Continuation continuation) { assert walk.isNonNull(); @@ -269,6 +270,15 @@ protected boolean advancePhysically() { return false; } + /* + * Store the StoredContinuation object in a field to avoid problems in case that the + * continuation continues execution in another thread in the meanwhile. + */ + stored = ContinuationInternals.getStoredContinuation(continuation); + if (stored == null) { + return false; + } + if (advancePhysically0()) { return true; } else { @@ -286,20 +296,19 @@ protected boolean advancePhysically() { @Uninterruptible(reason = "Prevent GC while in this method.") private boolean advancePhysically0() { - StoredContinuation stored = ContinuationInternals.getStoredContinuation(continuation); if (initialized) { /* * Because we are interruptible in between walking frames, pointers into the stored * continuation become invalid if a garbage collection moves the object. So, we need * to update all cached stack pointer values before we can continue the walk. */ - JavaStackWalker.updateStackPointer(walk, stored, 0); + JavaStackWalker.updateStackPointerForContinuation(walk, stored); } else { initialized = true; - JavaStackWalker.initialize(walk, stored); + JavaStackWalker.initializeForContinuation(walk, stored); } - if (!JavaStackWalker.advance(walk, stored)) { + if (!JavaStackWalker.advanceForContinuation(walk, stored)) { return false; } @@ -314,7 +323,7 @@ private boolean advancePhysically0() { Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); - /* This interruptible call may move the continuation. */ + /* This interruptible call might trigger a GC that moves continuation objects. */ CodeInfoQueryResult physicalFrame = queryCodeInfoInterruptibly(info, frame.getIP()); regularVFrame = physicalFrame.getFrameInfo(); } finally { @@ -326,6 +335,8 @@ private boolean advancePhysically0() { @Override protected void invalidate() { walk = WordFactory.nullPointer(); + continuation = null; + stored = null; } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java index ca62dfbcbefb..29b43be5f1da 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackWalker.java @@ -133,46 +133,11 @@ public static int walkCurrentThread(SamplerSampleWriterData data, CodePointer to * We might be in the middle of pushing a new frame anchor. In that case, the top * frame anchor will have invalid values and needs to be filtered out. */ - anchor = filterTopFrameAnchorIfInvalid(anchor, topFrameSP); + anchor = filterTopFrameAnchorIfIncomplete(anchor); /* Move SP to the top of the caller frame. */ long topFrameEncodedSize = CodeInfoAccess.lookupEncodedFrameSize(codeInfo, ip); boolean topFrameIsEntryPoint = CodeInfoQueryResult.isEntryPoint(topFrameEncodedSize); - if (isSPAligned(sp)) { - UnsignedWord topFrameSize = WordFactory.unsigned(CodeInfoQueryResult.getTotalFrameSize(topFrameEncodedSize)); - if (SubstrateOptions.hasFramePointer() && !hasValidCaller(sp, topFrameSize, topFrameIsEntryPoint, anchor)) { - /* - * If we have a frame pointer, then the stack pointer can be aligned while - * we are in the method prologue/epilogue (i.e., the frame pointer and the - * return address are on top of the stack, but the actual stack frame is - * missing). We should reach the caller if we skip the incomplete top frame - * (frame pointer and return address). - */ - sp = sp.add(FrameAccess.wordSize() * 2); - } else { - /* - * Stack looks walkable - skip the top frame as we already recorded the - * corresponding IP. - */ - sp = sp.add(topFrameSize); - } - } else { - /* - * The async sampler interrupted the thread while it was in the middle of - * manipulating the stack (e.g., in a method prologue/epilogue). Most likely, - * there is a valid return address at the top of the stack that we can just - * skip. - */ - sp = sp.add(FrameAccess.wordSize()); - } - - /* Do a basic sanity check and decide if it makes sense to continue. */ - assert isSPAligned(sp); - if (!isCallerSPValid(topFrameSP, sp)) { - /* One of the assumptions above was incorrect. */ - return UNPARSEABLE_STACK; - } - if (topFrameIsEntryPoint) { /* * If the top frame is for an entry point, then the caller needs to be treated @@ -189,10 +154,42 @@ public static int walkCurrentThread(SamplerSampleWriterData data, CodePointer to ip = anchor.getLastJavaIP(); anchor = anchor.getPreviousAnchor(); } else { - /* - * The top frame and its caller are probably Java frames. Read the return - * address. - */ + /* Both the top frame and its caller are probably Java frames. */ + if (isSPAligned(sp)) { + UnsignedWord topFrameSize = WordFactory.unsigned(CodeInfoQueryResult.getTotalFrameSize(topFrameEncodedSize)); + if (SubstrateOptions.hasFramePointer() && !hasValidCaller(sp, topFrameSize, topFrameIsEntryPoint, anchor)) { + /* + * If we have a frame pointer, then the stack pointer can be aligned + * while we are in the method prologue/epilogue (i.e., the frame pointer + * and the return address are on top of the stack, but the actual stack + * frame is missing). We should reach the caller if we skip the + * incomplete top frame (frame pointer and return address). + */ + sp = sp.add(FrameAccess.wordSize() * 2); + } else { + /* + * Stack looks walkable - skip the top frame as we already recorded the + * corresponding IP. + */ + sp = sp.add(topFrameSize); + } + } else { + /* + * The async sampler interrupted the thread while it was in the middle of + * manipulating the stack (e.g., in a method prologue/epilogue). Most + * likely, there is a valid return address at the top of the stack that we + * can just skip. + */ + sp = sp.add(FrameAccess.wordSize()); + } + + /* Do a basic sanity check and decide if it makes sense to continue. */ + assert isSPAligned(sp); + if (!isCallerSPValid(topFrameSP, sp)) { + /* One of the assumptions above was incorrect. */ + return UNPARSEABLE_STACK; + } + ip = FrameAccess.singleton().unsafeReadReturnAddress(sp); } } else { @@ -314,8 +311,8 @@ private static int computeHash(int oldHash, long ip) { } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static JavaFrameAnchor filterTopFrameAnchorIfInvalid(JavaFrameAnchor anchor, Pointer currentSP) { - if (anchor.isNonNull() && !isCallerValid(currentSP, anchor.getLastJavaSP(), anchor.getLastJavaIP())) { + private static JavaFrameAnchor filterTopFrameAnchorIfIncomplete(JavaFrameAnchor anchor) { + if (anchor.isNonNull() && (anchor.getLastJavaSP().isNull() || anchor.getLastJavaIP().isNull())) { /* We are probably in the middle of pushing a frame anchor, so filter the top anchor. */ return anchor.getPreviousAnchor(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java index 0819e88d2bd5..4def61b84719 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/ExceptionUnwind.java @@ -183,9 +183,9 @@ private static void defaultUnwindException(Pointer startSP, boolean fromMethodWi boolean hasCalleeSavedRegisters = fromMethodWithCalleeSavedRegisters; /* - * callerSP and startIP identify already the caller of the frame that wants to unwind an - * exception. So we can start looking for the exception handler immediately in that frame, - * without skipping any frames in between. + * callerSP identifies the caller of the frame that wants to unwind an exception. So we can + * start looking for the exception handler immediately in that frame, without skipping any + * frames in between. */ JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); JavaStackWalker.initialize(walk, thread, startSP); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java index 4d20f6a076e0..0dd5c78808ef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrameAnchors.java @@ -51,7 +51,7 @@ /** * Maintains the linked list of {@link JavaFrameAnchor} for stack walking. Note that a thread may - * only push/pop/modify a frame anchor, while its thread status is + * only push/pop/modify a frame anchor while its thread status is * {@link StatusSupport#STATUS_IN_JAVA}. This is necessary to guarantee that we can walk the stack * of other threads consistently while in a VM operation. */ @@ -67,8 +67,8 @@ public static void pushFrameAnchor(JavaFrameAnchor newAnchor) { /* * Set IP and SP to null during initialization, these values will later be overwritten by - * proper ones. The intention is to not see stale values when debugging or in signal - * handlers. + * proper ones (see usages of KnownOffsets.getJavaFrameAnchorLastSPOffset() in the backend). + * The intention is to not see stale values when debugging or in signal handlers. */ newAnchor.setLastJavaIP(WordFactory.nullPointer()); newAnchor.setLastJavaSP(WordFactory.nullPointer()); @@ -98,7 +98,7 @@ public static JavaFrameAnchor getFrameAnchor() { /** * Returns the last Java frame anchor for the given thread, or null if there is none. Note that * even at a safepoint, there is no guarantee that all stopped {@link IsolateThread}s have a - * Java frame anchor (e.g., threads that are currently attaching don't necesssarily have a frame + * Java frame anchor (e.g., threads that are currently attaching don't necessarily have a frame * anchor). */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java index a9dcfebdc4ac..0a970262effe 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalk.java @@ -45,8 +45,8 @@ public interface JavaStackWalk extends PointerBase { * The actual implementation. Most stack-walk related fields may only be accessed in * {@link JavaStackWalker}. * - * Note that this data structure stores some information about the current physical stack frame - * (e.g., SP, IP, frame size) and also some state that is only needed for the stack walk. + * Note that this data structure stores some information about the current physical stack frame (see + * {@link JavaFrame}) and also some state that is only needed for the stack walk. * * If interruptible code is executed while a stack walk is in progress, IP and code-related fields * in this data structure may contain stale/outdated values (code may get deoptimized). diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index 7845f3cd5886..b6e36bb221ba 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -80,16 +80,16 @@ * {@link JfrStackWalker}) needs to be changed as well. * *

- * Stack walking may skip frames of AOT or JIT-compiled code if the walked thread has C code on the - * stack that was called without a transition. In some situations, this can result in unexpected - * behavior. Note that signal handlers are also essentially C code that is called without a - * transition. Here is one example: + * Note that starting a stack walk is potentially dangerous when the walked thread has C code on the + * stack that was called without a transition (e.g., starting a stack walk in a signal handler). In + * such a case, stack walking may skip frames of AOT or JIT-compiled code, which can result in + * unexpected behavior. Here is one example: *

    *
  • We do a C call with {@link Transition#TO_NATIVE} (pushes a {@link JavaFrameAnchor}).
    • *
  • The C call finishes and we are back in AOT-compiled code.
    • *
  • The AOT-compiled code wants to do a transition back to {@link StatusSupport#STATUS_IN_JAVA} * but the fastpath fails.
    • - *
  • The thread calls the slowpath, which adds more frames for AOT-compiled code to the + *
  • The thread calls the slowpath, which pushes more frames of AOT-compiled code on the * stack.
    • *
  • A signal handler (such as the async sampler) interrupts execution and pushes native frames to * the stack.
    • @@ -119,8 +119,8 @@ public static int sizeOfJavaStackWalk() { /** * Returns information about the current physical Java frame. Note that this data is updated - * in-place when {@link #advance} is called. During a stack walk, it is therefore not possible - * to access the data of a previous frame. + * in-place when {@link #continueStackWalk} is called. During a stack walk, it is therefore not + * possible to access the data of a previous frame. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static JavaFrame getCurrentFrame(JavaStackWalk walk) { @@ -176,7 +176,7 @@ public static void initialize(JavaStackWalk walk, IsolateThread thread, Pointer } @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) - public static void initialize(JavaStackWalk walk, StoredContinuation continuation) { + public static void initializeForContinuation(JavaStackWalk walk, StoredContinuation continuation) { assert continuation != null; CodePointer startIP = StoredContinuationAccess.getIP(continuation); @@ -191,7 +191,7 @@ public static void initialize(JavaStackWalk walk, StoredContinuation continuatio } @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) - public static void initialize(JavaStackWalk walk, StoredContinuation continuation, CodePointer startIP) { + public static void initializeForContinuation(JavaStackWalk walk, StoredContinuation continuation, CodePointer startIP) { assert continuation != null; assert startIP.isNonNull(); @@ -246,9 +246,9 @@ private static void initWalk0(JavaStackWalk walk, Pointer startSP, Pointer endSP } @Uninterruptible(reason = "JavaStackWalk must not contain stale values when this method returns.", callerMustBe = true) - public static void updateStackPointer(JavaStackWalk walk, StoredContinuation continuation, int startSPOffset) { + public static void updateStackPointerForContinuation(JavaStackWalk walk, StoredContinuation continuation) { JavaStackWalkImpl w = cast(walk); - Pointer newStartSP = StoredContinuationAccess.getFramesStart(continuation).add(startSPOffset); + Pointer newStartSP = StoredContinuationAccess.getFramesStart(continuation); long delta = newStartSP.rawValue() - w.getStartSP().rawValue(); long newEndSP = w.getEndSP().rawValue() + delta; @@ -264,21 +264,21 @@ public static void updateStackPointer(JavaStackWalk walk, StoredContinuation con @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) public static boolean advance(JavaStackWalk walk, IsolateThread thread) { - return advance(walk, thread, null); + return advance0(walk, thread, null); } @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) - public static boolean advance(JavaStackWalk walk, StoredContinuation continuation) { - return advance(walk, WordFactory.nullPointer(), continuation); + public static boolean advanceForContinuation(JavaStackWalk walk, StoredContinuation continuation) { + return advance0(walk, WordFactory.nullPointer(), continuation); } @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) - private static boolean advance(JavaStackWalk walk, IsolateThread thread, StoredContinuation continuation) { + private static boolean advance0(JavaStackWalk walk, IsolateThread thread, StoredContinuation continuation) { JavaStackWalkImpl w = cast(walk); if (!w.getStarted()) { return startStackWalk(w, thread, continuation); } - return advance0(w, thread, continuation); + return continueStackWalk(w, thread, continuation); } @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) @@ -314,7 +314,7 @@ private static boolean startStackWalk(JavaStackWalkImpl walk, IsolateThread thre } @Uninterruptible(reason = "Prevent deoptimization and GC while in this method.", callerMustBe = true) - private static boolean advance0(JavaStackWalkImpl walk, IsolateThread thread, StoredContinuation continuation) { + private static boolean continueStackWalk(JavaStackWalkImpl walk, IsolateThread thread, StoredContinuation continuation) { assert thread.isNull() != (continuation == null); JavaFrame frame = getCurrentFrame(walk); @@ -454,12 +454,12 @@ static boolean doWalk(JavaStackWalk walk, IsolateThread thread, ParameterizedSta CodePointer ip = frame.getIP(); if (JavaFrames.isUnknownFrame(frame)) { - return callUnknownFrameVisitor(sp, ip, visitor, data); + return visitUnknownFrame(sp, ip, visitor, data); } DeoptimizedFrame deoptimizedFrame = Deoptimizer.checkDeoptimized(frame); if (deoptimizedFrame != null) { - if (!callDeoptimizedFrameVisitor(sp, ip, deoptimizedFrame, visitor, data)) { + if (!vistDeoptimizedFrame(sp, ip, deoptimizedFrame, visitor, data)) { return false; } } else { @@ -467,7 +467,7 @@ static boolean doWalk(JavaStackWalk walk, IsolateThread thread, ParameterizedSta Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); - if (!callRegularFrameVisitor(sp, ip, info, visitor, data)) { + if (!visitRegularFrame(sp, ip, info, visitor, data)) { return false; } } finally { @@ -480,19 +480,19 @@ static boolean doWalk(JavaStackWalk walk, IsolateThread thread, ParameterizedSta } @Uninterruptible(reason = "CodeInfo in JavaStackWalk is currently null, and we are going to abort the stack walking.", calleeMustBe = false) - private static boolean callUnknownFrameVisitor(Pointer sp, CodePointer ip, ParameterizedStackFrameVisitor visitor, Object data) { + private static boolean visitUnknownFrame(Pointer sp, CodePointer ip, ParameterizedStackFrameVisitor visitor, Object data) { return visitor.unknownFrame(sp, ip, data); } @Uninterruptible(reason = "Wraps the now safe call to the possibly interruptible visitor.", callerMustBe = true, calleeMustBe = false) @RestrictHeapAccess(reason = "Whitelisted because some StackFrameVisitor implementations can allocate.", access = RestrictHeapAccess.Access.UNRESTRICTED) - private static boolean callRegularFrameVisitor(Pointer sp, CodePointer ip, CodeInfo info, ParameterizedStackFrameVisitor visitor, Object data) { + private static boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo info, ParameterizedStackFrameVisitor visitor, Object data) { return visitor.visitRegularFrame(sp, ip, info, data); } @Uninterruptible(reason = "Wraps the now safe call to the possibly interruptible visitor.", callerMustBe = true, calleeMustBe = false) @RestrictHeapAccess(reason = "Whitelisted because some StackFrameVisitor implementations can allocate.", access = RestrictHeapAccess.Access.UNRESTRICTED) - private static boolean callDeoptimizedFrameVisitor(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, ParameterizedStackFrameVisitor visitor, Object data) { + private static boolean vistDeoptimizedFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, ParameterizedStackFrameVisitor visitor, Object data) { return visitor.visitDeoptimizedFrame(sp, ip, deoptimizedFrame, data); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java index 73f49219019e..6e5141671600 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ParameterizedStackFrameVisitor.java @@ -50,7 +50,7 @@ public abstract class ParameterizedStackFrameVisitor { * @param sp The stack pointer of the frame being visited. * @param ip The instruction pointer of the frame being visited. * @param codeInfo Information on the code at the IP, for use with {@link CodeInfoAccess}. - * @param data An arbitrary data value passed through the stack walker. + * @param data An implementation-provided object that is passed through the stack walker. * @return true if visiting should continue, false otherwise. */ protected abstract boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data); @@ -61,7 +61,7 @@ public abstract class ParameterizedStackFrameVisitor { * @param originalSP The stack pointer to the physical (already invalidated) stack frame. * @param deoptStubIP The instruction pointer for the deopt stub. * @param deoptimizedFrame The deoptimized frame. - * @param data An arbitrary data value passed through the stack walker. + * @param data An implementation-provided object that is passed through the stack walker. * @return true if visiting should continue, false otherwise. */ protected abstract boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data); @@ -74,7 +74,7 @@ public abstract class ParameterizedStackFrameVisitor { * * @param sp The stack pointer of the frame being visited. * @param ip The instruction pointer of the frame being visited. - * @param data An arbitrary data value passed through the stack walker. + * @param data An implementation-provided object that is passed through the stack walker. * @return The value returned to the caller of stack walking. Note that walking of the thread is * always aborted, regardless of the return value. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java index 79ac3a36ae50..b701ae117d9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/ThreadStackPrinter.java @@ -233,6 +233,8 @@ public static boolean printStacktrace(IsolateThread thread, Pointer initialSP, C if (anchor.isNonNull()) { sp = anchor.getLastJavaSP(); ip = anchor.getLastJavaIP(); + } else { + return false; } }