diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index bee7b2871e1b..1f8e4b4308fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { assert VMOperation.isGCInProgress() : "use only during GC"; guaranteeSizeParametersInitialized(); @@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index bc15be90bdcf..86dd5c3e0c48 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static public interface AlignedHeader extends HeapChunk.Header { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize); } @@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) { } /** Allocate uninitialized memory within this AlignedHeapChunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { Pointer result = WordFactory.nullPointer(); UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -129,6 +131,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { } /** Return the offset of an object within the objects part of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) { Pointer objectsStart = getObjectsStart(that); return objectPointer.subtract(objectsStart); @@ -139,6 +142,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java index 2d18a356ece7..3ac40d5a385c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java @@ -48,6 +48,7 @@ static AuxiliaryImageHeap singleton() { boolean walkObjects(ObjectVisitor visitor); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean walkRegions(MemoryWalker.ImageHeapRegionVisitor visitor); ImageHeapInfo getImageHeapInfo(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java index ae89d3740679..150aa514aac1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; @@ -49,7 +50,7 @@ static int getMaxSurvivorSpaces(Integer userValue) { private BasicCollectionPolicies() { } - abstract static class BasicPolicy implements CollectionPolicy { + public abstract static class BasicPolicy implements CollectionPolicy { protected static UnsignedWord m(long bytes) { assert 0 <= bytes; return WordFactory.unsigned(bytes).multiply(1024).multiply(1024); @@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { return WordFactory.zero(); } @@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { return 1; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 17c24b576832..68c036cb640b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -54,12 +54,14 @@ final class ChunksAccounting { reset(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void reset() { alignedCount = 0L; unalignedCount = 0L; unalignedChunkBytes = WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getChunkBytes() { return getAlignedChunkBytes().add(getUnalignedChunkBytes()); } @@ -96,6 +98,7 @@ void noteAlignedHeapChunk() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteAlignedHeapChunk() { alignedCount--; if (parent != null) { @@ -117,10 +120,12 @@ private void noteUnaligned(UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void unnoteUnaligned(UnsignedWord size) { unalignedCount--; unalignedChunkBytes = unalignedChunkBytes.subtract(size); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 1d11025ef0f7..ba738c3df223 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; @@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x * this amount can be used for surviving objects. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnsignedWord getSurvivorSpacesCapacity(); /** The capacity of the young generation, comprising the eden and survivor spaces. */ @@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of * survivor spaces + 1}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getTenuringAge(); /** Called at the beginning of a collection, in the safepoint operation. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java index 3515aab4fb29..e991441a0f15 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java @@ -48,12 +48,12 @@ public CompleteGarbageCollectorMXBean() { @Override public long getCollectionCount() { - return HeapImpl.getHeapImpl().getGCImpl().getAccounting().getCompleteCollectionCount(); + return HeapImpl.getGCImpl().getAccounting().getCompleteCollectionCount(); } @Override public long getCollectionTime() { - long nanos = HeapImpl.getHeapImpl().getGCImpl().getAccounting().getCompleteCollectionTotalNanos(); + long nanos = HeapImpl.getGCImpl().getAccounting().getCompleteCollectionTotalNanos(); return TimeUtils.roundNanosToMillis(nanos); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 128df06b6087..c006ab6faa54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) { /** Called after an object has been promoted from the young generation to the old generation. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 6dd5aff78aa9..e998229ffe5a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -44,7 +44,6 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.RuntimeAssertionsSupport; import com.oracle.svm.core.SubstrateGCOptions; @@ -100,7 +99,6 @@ public final class GCImpl implements GC { private final GreyToBlackObjRefVisitor greyToBlackObjRefVisitor = new GreyToBlackObjRefVisitor(); private final GreyToBlackObjectVisitor greyToBlackObjectVisitor = new GreyToBlackObjectVisitor(greyToBlackObjRefVisitor); - private final BlackenImageHeapRootsVisitor blackenImageHeapRootsVisitor = new BlackenImageHeapRootsVisitor(); private final RuntimeCodeCacheWalker runtimeCodeCacheWalker = new RuntimeCodeCacheWalker(greyToBlackObjRefVisitor); private final RuntimeCodeCacheCleaner runtimeCodeCacheCleaner = new RuntimeCodeCacheCleaner(); @@ -515,7 +513,7 @@ static boolean runtimeAssertions() { @Fold public static GCImpl getGCImpl() { - GCImpl gcImpl = HeapImpl.getHeapImpl().getGCImpl(); + GCImpl gcImpl = HeapImpl.getGCImpl(); assert gcImpl != null; return gcImpl; } @@ -525,6 +523,7 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } @@ -538,11 +537,7 @@ private void scavenge(boolean incremental) { try { startTicks = JfrGCEvents.startGCPhasePause(); try { - if (incremental) { - cheneyScanFromDirtyRoots(); - } else { - cheneyScanFromRoots(); - } + cheneyScan(incremental); } finally { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), incremental ? "Incremental Scan" : "Scan", startTicks); } @@ -620,6 +615,7 @@ private void scavenge(boolean incremental) { * compiled code to the Java heap must be consider as either strong or weak references, * depending on whether the code is currently on the execution stack. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkRuntimeCodeCache() { Timer walkRuntimeCodeCacheTimer = timers.walkRuntimeCodeCache.open(); try { @@ -638,6 +634,16 @@ private void cleanRuntimeCodeCache() { } } + @Uninterruptible(reason = "We don't want any safepoint checks in the core part of the GC.") + private void cheneyScan(boolean incremental) { + if (incremental) { + cheneyScanFromDirtyRoots(); + } else { + cheneyScanFromRoots(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void cheneyScanFromRoots() { Timer cheneyScanFromRootsTimer = timers.cheneyScanFromRoots.open(); try { @@ -707,6 +713,7 @@ private void cheneyScanFromRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { @@ -798,6 +805,7 @@ private void cheneyScanFromDirtyRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promoteChunksWithPinnedObjects() { Timer promotePinnedObjectsTimer = timers.promotePinnedObjects.open(); try { @@ -821,6 +829,7 @@ private void promoteChunksWithPinnedObjects() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) { PinnedObjectImpl firstOpen = null; PinnedObjectImpl lastOpen = null; @@ -849,7 +858,7 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) @NeverInline("Starting a stack walk in the caller frame. " + "Note that we could start the stack frame also further down the stack, because GC stack frames must not access any objects that are processed by the GC. " + "But we don't store stack frame information for the first frame we would need to process.") - @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) + @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.") private void blackenStackRoots() { Timer blackenStackRootsTimer = timers.blackenStackRoots.open(); try { @@ -891,7 +900,7 @@ private void blackenStackRoots() { * {@link SimpleCodeInfoQueryResult} twice per frame, and also ensures that there are no virtual * calls to a stack frame visitor. */ - @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) + @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.") private void walkStack(JavaStackWalk walk) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; @@ -943,6 +952,7 @@ private void walkStack(JavaStackWalk walk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); @@ -956,6 +966,7 @@ private void walkThreadLocals() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyImageHeapRoots() { if (!HeapImpl.usesImageHeapCardMarking()) { blackenImageHeapRoots(); @@ -978,6 +989,7 @@ private void blackenDirtyImageHeapRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, UnalignedHeader firstUnaligned) { /* * We clean and remark cards of the image heap only during complete collections when we also @@ -999,6 +1011,7 @@ private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, Unalign } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenImageHeapRoots() { if (HeapImpl.usesImageHeapCardMarking()) { // Avoid scanning the entire image heap even for complete collections: its remembered @@ -1009,22 +1022,25 @@ private void blackenImageHeapRoots() { Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open(); try { - HeapImpl.getHeapImpl().walkNativeImageHeapRegions(blackenImageHeapRootsVisitor); + blackenImageHeapRoots(HeapImpl.getImageHeapInfo()); + if (AuxiliaryImageHeap.isPresent()) { + ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); + if (auxImageHeapInfo != null) { + blackenImageHeapRoots(auxImageHeapInfo); + } + } } finally { blackenImageHeapRootsTimer.close(); } } - private class BlackenImageHeapRootsVisitor implements MemoryWalker.ImageHeapRegionVisitor { - @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { - if (access.containsReferences(region) && access.isWritable(region)) { - access.visitObjects(region, greyToBlackObjectVisitor); - } - return true; - } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableReferenceObject, imageHeapInfo.lastWritableReferenceObject, greyToBlackObjectVisitor, true); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableHugeObject, imageHeapInfo.lastWritableHugeObject, greyToBlackObjectVisitor, false); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyCardRoots() { Timer blackenDirtyCardRootsTimer = timers.blackenDirtyCardRoots.open(); try { @@ -1039,6 +1055,7 @@ private void blackenDirtyCardRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void prepareForPromotion(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); heap.getOldGeneration().prepareForPromotion(); @@ -1047,6 +1064,7 @@ private static void prepareForPromotion(boolean isIncremental) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { @@ -1060,6 +1078,7 @@ private void scanGreyObjects(boolean isIncremental) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void scanGreyObjectsLoop() { HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); @@ -1072,6 +1091,7 @@ private static void scanGreyObjectsLoop() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); @@ -1105,6 +1125,7 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1113,6 +1134,7 @@ private static Header getChunk(Object obj, boolean isAligned) { return UnalignedHeapChunk.getEnclosingChunk(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promotePinnedObject(PinnedObjectImpl pinned) { HeapImpl heap = HeapImpl.getHeapImpl(); Object referent = pinned.getObject(); @@ -1217,6 +1239,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } + @Fold GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } @@ -1266,7 +1289,7 @@ protected void operate(NativeVMOperationData data) { */ ImplicitExceptions.activateImplicitExceptionsAreFatal(); try { - HeapImpl.getHeapImpl().getGCImpl().collectOperation((CollectionVMOperationData) data); + HeapImpl.getGCImpl().collectOperation((CollectionVMOperationData) data); } catch (Throwable t) { throw VMError.shouldNotReachHere(t); } finally { @@ -1277,7 +1300,7 @@ protected void operate(NativeVMOperationData data) { @Override protected boolean hasWork(NativeVMOperationData data) { CollectionVMOperationData d = (CollectionVMOperationData) data; - return HeapImpl.getHeapImpl().getGCImpl().getCollectionEpoch().equal(d.getRequestingEpoch()); + return HeapImpl.getGCImpl().getCollectionEpoch().equal(d.getRequestingEpoch()); } } @@ -1326,6 +1349,7 @@ public boolean isEmpty() { return firstAligned.isNull() && firstUnaligned.isNull(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void add(AlignedHeader chunks) { if (chunks.isNonNull()) { assert HeapChunk.getPrevious(chunks).isNull() : "prev must be null"; @@ -1338,6 +1362,7 @@ public void add(AlignedHeader chunks) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void add(UnalignedHeader chunks) { if (chunks.isNonNull()) { assert HeapChunk.getPrevious(chunks).isNull() : "prev must be null"; @@ -1361,6 +1386,7 @@ void release(boolean keepAllAlignedChunks) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static > T getLast(T chunks) { T prev = chunks; T next = HeapChunk.getNext(prev); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 35b694891c19..e487a4c7f1d2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,6 +27,8 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -66,6 +68,8 @@ public String getName() { * promotion was done by copying, or {@code null} if there was insufficient capacity in * this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); /** @@ -79,6 +83,8 @@ public String getName() { * was promoted through HeapChunk motion, or {@code null} if there was insufficient * capacity in this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 3435527236ac..09ad12fd3568 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -31,7 +31,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.util.VMError; /** @@ -57,26 +57,25 @@ final class GreyObjectsWalker { * Take a snapshot of a Space, such that all Objects in the Space are now black, and any new * Objects in the Space will be grey, and can have an ObjectVisitor applied to them. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setScanStart(Space s) { - Log trace = Log.noopLog().string("[Space.GreyObjectsWalker.setScanStart:").string(" s: ").string(s.getName()); space = s; AlignedHeapChunk.AlignedHeader aChunk = s.getLastAlignedHeapChunk(); alignedHeapChunk = aChunk; - trace.string(" alignedHeapChunk: ").zhex(alignedHeapChunk).string(" isNull: ").bool(aChunk.isNull()); alignedTop = (aChunk.isNonNull() ? HeapChunk.getTopPointer(aChunk) : WordFactory.nullPointer()); - trace.string(" alignedTop: ").zhex(alignedTop); unalignedHeapChunk = s.getLastUnalignedHeapChunk(); - trace.string(" unalignedChunkPointer: ").zhex(unalignedHeapChunk).string("]").newline(); } /** Compare the snapshot to the current state of the Space to see if there are grey Objects. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean haveGreyObjects() { return alignedHeapChunk.notEqual(space.getLastAlignedHeapChunk()) || alignedHeapChunk.isNonNull() && alignedTop.notEqual(HeapChunk.getTopPointer(alignedHeapChunk)) || unalignedHeapChunk.notEqual(space.getLastUnalignedHeapChunk()); } @NeverInline("Split the GC into reasonable compilation units") + @Uninterruptible(reason = "Called from uninterruptible code.") void walkGreyObjects() { while (haveGreyObjects()) { walkAlignedGreyObjects(); @@ -85,6 +84,7 @@ void walkGreyObjects() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { @@ -113,6 +113,7 @@ private void walkAlignedGreyObjects() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index ac1be2f4d2f8..3281a53ac41c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -30,6 +30,7 @@ import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -59,12 +60,14 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { return visitObjectReferenceInline(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); @@ -131,16 +134,22 @@ public interface Counters extends AutoCloseable { @Override void close(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteObjRef(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNullReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteForwardedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNonHeapReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteCopiedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); void toLog(); @@ -185,31 +194,37 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { objRef += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { nullReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { forwardedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { nonHeapReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { copiedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { unmodifiedReference += 1L; } @@ -244,26 +259,32 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 9a8b48845d46..bc432b866261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; @@ -55,6 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 6014037a1098..02512d65b1e1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -177,6 +177,7 @@ public interface Header> extends HeaderPadding { void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); @@ -306,11 +307,12 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { + if (!callVisitor(visitor, obj)) { return false; } offset = offset.add(LayoutEncoding.getSizeFromObjectInlineInGC(obj)); @@ -318,6 +320,11 @@ public static boolean walkObjectsFromInline(Header that, Pointer startOffset, return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectVisitor visitor, Object obj) { + return visitor.visitObjectInline(obj); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord availableObjectMemory(Header that) { return that.getEndOffset().subtract(that.getTopOffset()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 4df6532b8e51..f94172566a49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -35,11 +34,11 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; @@ -89,23 +88,15 @@ public UnsignedWord getBytesInUnusedChunks() { return bytesInUnusedAlignedChunks.get(); } - @AlwaysInline("Remove all logging when noopLog is returned by this method") - private static Log log() { - return Log.noopLog(); - } - private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk"); private static final OutOfMemoryError UNALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an unaligned heap chunk"); /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); - log().string("[HeapChunkProvider.produceAlignedChunk chunk size: ").unsigned(chunkSize).newline(); - AlignedHeader result = popUnusedAlignedChunk(); - log().string(" unused chunk: ").zhex(result).newline(); - if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ noteFirstAllocationTime(); @@ -113,7 +104,6 @@ AlignedHeader produceAlignedChunk() { if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } - log().string(" new chunk: ").zhex(result).newline(); AlignedHeapChunk.initialize(result, chunkSize); } @@ -123,8 +113,6 @@ AlignedHeader produceAlignedChunk() { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" result chunk: ").zhex(result).string(" ]").newline(); return result; } @@ -200,13 +188,10 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { if (SubstrateOptions.MultiThreaded.getValue()) { VMThreads.guaranteeOwnsThreadMutex("Should hold the lock when pushing to the global list."); } - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); HeapChunk.setNext(chunk, unusedAlignedChunks.get()); unusedAlignedChunks.set(chunk); bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize()); - - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); } /** @@ -218,15 +203,13 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { * garbage collections, I avoid the ABA problem by making the kernel of this method * uninterruptible so it can not be interrupted by a safepoint. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeader popUnusedAlignedChunk() { - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); - AlignedHeader result = popUnusedAlignedChunkUninterruptibly(); if (result.isNull()) { return WordFactory.nullPointer(); } else { bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize()); - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); return result; } } @@ -268,7 +251,6 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { /** Acquire an UnalignedHeapChunk from the operating system. */ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - log().string("[HeapChunkProvider.produceUnalignedChunk objectSize: ").unsigned(objectSize).string(" chunkSize: ").zhex(chunkSize).newline(); noteFirstAllocationTime(); UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); @@ -282,8 +264,6 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" returns ").zhex(result).string(" ]").newline(); return result; } @@ -300,10 +280,10 @@ static void consumeUnalignedChunks(UnalignedHeader firstChunk) { freeUnalignedChunkList(firstChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void zap(Header chunk, WordBase value) { Pointer start = HeapChunk.getTopPointer(chunk); Pointer limit = HeapChunk.getEndPointer(chunk); - log().string(" zap chunk: ").zhex(chunk).string(" start: ").zhex(start).string(" limit: ").zhex(limit).string(" value: ").zhex(value).newline(); for (Pointer p = start; p.belowThan(limit); p = p.add(FrameAccess.wordSize())) { p.writeWord(0, value); } @@ -332,12 +312,14 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return continueVisiting; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void noteFirstAllocationTime() { if (firstAllocationTime == 0L) { firstAllocationTime = System.nanoTime(); } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) long getFirstAllocationTime() { return firstAllocationTime; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index b5f84cedaaad..e7fa7e9e2940 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -192,12 +192,6 @@ public boolean walkObjects(ObjectVisitor visitor) { return walkImageHeapObjects(visitor) && walkCollectedHeapObjects(visitor); } - /** Walk the regions of the heap. */ - boolean walkMemory(MemoryWalker.Visitor visitor) { - VMOperation.guaranteeInProgressAtSafepoint("must only be executed at a safepoint"); - return walkNativeImageHeapRegions(visitor) && getYoungGeneration().walkHeapChunks(visitor) && getOldGeneration().walkHeapChunks(visitor) && getChunkProvider().walkHeapChunks(visitor); - } - /** Tear down the heap and release its memory. */ @Override @Uninterruptible(reason = "Tear-down in progress.") @@ -214,14 +208,20 @@ public ObjectHeader getObjectHeader() { return objectHeaderImpl; } - ObjectHeaderImpl getObjectHeaderImpl() { - return objectHeaderImpl; + @Fold + static ObjectHeaderImpl getObjectHeaderImpl() { + return getHeapImpl().objectHeaderImpl; } @Fold @Override public GC getGC() { - return getGCImpl(); + return getHeapImpl().gcImpl; + } + + @Fold + static GCImpl getGCImpl() { + return getHeapImpl().gcImpl; } @Fold @@ -235,10 +235,6 @@ public HeapAccounting getAccounting() { return accounting; } - GCImpl getGCImpl() { - return gcImpl; - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { @@ -262,6 +258,7 @@ public OldGeneration getOldGeneration() { return oldGeneration; } + @Fold AtomicReference getPinHead() { return pinHead; } @@ -468,11 +465,6 @@ public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { return getYoungGeneration().walkObjects(visitor) && getOldGeneration().walkObjects(visitor); } - boolean walkNativeImageHeapRegions(MemoryWalker.ImageHeapRegionVisitor visitor) { - return ImageHeapWalker.walkRegions(imageHeapInfo, visitor) && - (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkRegions(visitor)); - } - @Override public void doReferenceHandling() { if (ReferenceHandler.isExecutedManually()) { @@ -704,7 +696,7 @@ public void dirtyAllReferencesOf(Object obj) { @Override public long getMillisSinceLastWholeHeapExamined() { - return getGCImpl().getMillisSinceLastWholeHeapExamined(); + return HeapImpl.getGCImpl().getMillisSinceLastWholeHeapExamined(); } @Override @@ -717,6 +709,7 @@ public long getIdentityHashSalt(Object obj) { return HeapChunk.getIdentityHashSalt(chunk).rawValue(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer getImageHeapStart() { int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace(); if (imageHeapOffsetInAddressSpace > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 9c2be3f9a442..e5817b56068d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -137,6 +137,7 @@ public static UnsignedWord getLargeArrayThreshold() { * Zapping */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index 44d04a7cdd15..4be1beea5a6b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -30,10 +30,10 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.heap.UnknownObjectField; -import com.oracle.svm.core.heap.UnknownPrimitiveField; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.heap.UnknownPrimitiveField; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.snippets.KnownIntrinsics; @@ -187,10 +187,12 @@ public boolean isInImageHeap(Pointer objectPointer) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public AlignedHeader getFirstWritableAlignedChunk() { return asImageHeapChunk(offsetOfFirstWritableAlignedChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnalignedHeader getFirstWritableUnalignedChunk() { return asImageHeapChunk(offsetOfFirstWritableUnalignedChunk); } @@ -204,6 +206,7 @@ private static Pointer getObjectEnd(Object obj) { } @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static > T asImageHeapChunk(long offsetInImageHeap) { if (offsetInImageHeap < 0) { return (T) WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index 2c17d3bbf737..9f781524dcd8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -31,8 +31,9 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.os.CommittedMemoryProvider; @@ -75,11 +76,13 @@ static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisito } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) { if (firstObject == null || lastObject == null) { assert firstObject == null && lastObject == null; @@ -113,10 +116,10 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject while (current.belowOrEqual(limit)) { Object currentObject = current.toObject(); if (inlineObjectVisit) { - if (!visitor.visitObjectInline(currentObject)) { + if (!visitObjectInline(visitor, currentObject)) { return false; } - } else if (!visitor.visitObject(currentObject)) { + } else if (!visitObject(visitor, currentObject)) { return false; } current = LayoutEncoding.getImageHeapObjectEnd(current.toObject()); @@ -130,6 +133,16 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject } while (current.belowOrEqual(lastPointer)); return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean visitObject(ObjectVisitor visitor, Object currentObject) { + return visitor.visitObject(currentObject); + } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean visitObjectInline(ObjectVisitor visitor, Object currentObject) { + return visitor.visitObjectInline(currentObject); + } } abstract class MemoryWalkerAccessBase implements MemoryWalker.NativeImageHeapRegionAccess { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java index 7c08b40086b3..f11508e80d08 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java @@ -48,12 +48,12 @@ public IncrementalGarbageCollectorMXBean() { @Override public long getCollectionCount() { - return HeapImpl.getHeapImpl().getGCImpl().getAccounting().getIncrementalCollectionCount(); + return HeapImpl.getGCImpl().getAccounting().getIncrementalCollectionCount(); } @Override public long getCollectionTime() { - long nanos = HeapImpl.getHeapImpl().getGCImpl().getAccounting().getIncrementalCollectionTotalNanos(); + long nanos = HeapImpl.getGCImpl().getAccounting().getIncrementalCollectionTotalNanos(); return TimeUtils.roundNanosToMillis(nanos); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 47126cddf9e2..6a2a962d5aeb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -54,11 +54,13 @@ class JfrGCEventSupport { this.gcName = gcName; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long startGCPhasePause() { pushPhase(); return JfrTicks.elapsedTicks(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int stopGCPhasePause() { return popPhase(); } @@ -124,6 +126,7 @@ private static JfrEvent getGCPhasePauseEvent(int level) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void pushPhase() { assert currentPhase < MAX_PHASE_LEVEL; currentPhase++; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java index 63eb13e1ee81..9bcd47d1cac3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java @@ -28,9 +28,11 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; class JfrGCEvents { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long startGCPhasePause() { if (hasJfrSupport()) { return jfrSupport().startGCPhasePause(); @@ -38,6 +40,7 @@ public static long startGCPhasePause() { return 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void emitGCPhasePauseEvent(UnsignedWord gcEpoch, String name, long start) { if (hasJfrSupport()) { int level = jfrSupport().stopGCPhasePause(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 85a64165988b..be0def8590b7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -95,7 +95,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { } else { VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); numReservedBits = numMinimumReservedBits + 2; - VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(), "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); } numReservedExtraBits = numReservedBits - numAlignmentBits; @@ -104,7 +104,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { @Fold public static ObjectHeaderImpl getObjectHeaderImpl() { - ObjectHeaderImpl oh = HeapImpl.getHeapImpl().getObjectHeaderImpl(); + ObjectHeaderImpl oh = HeapImpl.getObjectHeaderImpl(); assert oh != null; return oh; } @@ -166,6 +166,7 @@ public boolean hasOptionalIdentityHashField(Word header) { return header.and(IDHASH_STATE_BITS).equal(inFieldState); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setIdentityHashInField(Object o) { assert VMOperation.isGCInProgress(); VMError.guarantee(!hasFixedIdentityHashField()); @@ -297,8 +298,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition) { - ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { if (partition.isWritable()) { header |= REMEMBERED_SET_BIT.rawValue(); } @@ -320,6 +320,7 @@ public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } @@ -335,12 +336,14 @@ public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setRememberedSetBit(Object o) { UnsignedWord oldHeader = readHeaderFromObject(o); UnsignedWord newHeader = oldHeader.or(REMEMBERED_SET_BIT); writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } @@ -356,14 +359,16 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (hasShift()) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -380,24 +385,33 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); - UnsignedWord forwardHeader; + UnsignedWord forwardHeader = getForwardHeader(copy); + ObjectAccess.writeLong(original, getHubOffset(), forwardHeader.rawValue()); + assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private UnsignedWord getForwardHeader(Object copy) { + UnsignedWord result; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + UnsignedWord compressedCopy = ReferenceAccess.singleton().getCompressedRepresentation(copy); + if (hasShift()) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + result = compressedCopy.shiftLeft(32).or(WordFactory.unsigned(0x00000000e0e0e0e0L)); } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + result = compressedCopy; } } else { - forwardHeader = Word.objectToUntrackedPointer(copy); + result = Word.objectToUntrackedPointer(copy); } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + + assert getHeaderBitsFromHeader(result).equal(0); + return result.or(FORWARDED_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -407,6 +421,11 @@ private UnsignedWord getHeaderBitsFromHeader(UnsignedWord header) { return header.and(reservedBitsMask); } + @Fold + static boolean hasShift() { + return ReferenceAccess.singleton().getCompressEncoding().hasShift(); + } + @Fold static boolean hasFixedIdentityHashField() { return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 395e39ef03fe..8c77d261b9b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -31,8 +31,8 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -73,6 +73,7 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -80,6 +81,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -88,6 +90,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { assert originalSpace.isFromSpace(); if (isAligned) { @@ -102,10 +105,12 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { getFromSpace().releaseChunks(chunkReleaser); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; @@ -139,6 +144,7 @@ void swapSpaces() { } /* Extract all the HeapChunks from FromSpace and append them to ToSpace. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void emptyFromSpaceIntoToSpace() { getToSpace().absorb(getFromSpace()); } @@ -158,13 +164,13 @@ UnsignedWord getChunkBytes() { return fromBytes.add(toBytes); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") AlignedHeapChunk.AlignedHeader requestAlignedChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - Log.log().string("[! OldGeneration.requestAlignedChunk: failure to allocate aligned chunk!]"); - throw VMError.shouldNotReachHere("Promotion failure"); + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } RememberedSet.get().enableRememberedSetForChunk(chunk); return chunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index dc945e0be145..9f85d0afa7bb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -77,12 +77,14 @@ static void pushPinnedObject(PinnedObjectImpl newHead) { } while (!pinHead.compareAndSet(sampleHead, newHead)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static PinnedObjectImpl getPinnedObjects() { assert VMOperation.isGCInProgress(); UninterruptibleUtils.AtomicReference pinHead = HeapImpl.getHeapImpl().getPinHead(); return pinHead.get(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void setPinnedObjects(PinnedObjectImpl list) { assert VMOperation.isGCInProgress(); UninterruptibleUtils.AtomicReference pinHead = HeapImpl.getHeapImpl().getPinHead(); @@ -108,6 +110,7 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getObject() { assert open : "Should not call getObject() on a closed PinnedObject."; return referent; @@ -133,14 +136,17 @@ public T addressOfArrayElement(int index) { return (T) addressOfObject().add(offsetOfArrayElement); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isOpen() { return open; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public PinnedObjectImpl getNext() { return next; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setNext(PinnedObjectImpl value) { // Avoid useless writes as those would dirty the card table unnecessarily. if (value != next) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index e3a4ee9b894b..e2b2df83407e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,6 +36,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; @@ -79,6 +80,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,6 +89,8 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header @@ -218,6 +222,7 @@ private static boolean processRememberedRef(Reference dr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); UnsignedWord header = ObjectHeader.readHeaderFromPointer(referentAddr); @@ -229,6 +234,7 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 1aff988d39a2..281dcbfc8e84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -94,6 +94,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isEmpty() { return (getFirstAlignedHeapChunk().isNull() && getFirstUnalignedHeapChunk().isNull()); } @@ -109,10 +110,12 @@ boolean isEdenSpace() { return age == 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isSurvivorSpace() { return age > 0 && age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -122,14 +125,17 @@ public boolean isOldSpace() { return age == (HeapParameters.getMaxSurvivorSpaces() + 1); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getAge() { return age; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getNextAgeForPromotion() { return age + 1; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isFromSpace() { return isFromSpace; } @@ -168,6 +174,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ @@ -182,6 +189,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return allocateInNewChunk(objectSize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (newChunk.isNonNull()) { @@ -190,6 +198,7 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); @@ -229,6 +238,7 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { assert VMOperation.isGCInProgress() : "Should only be called by the collector."; extractAlignedHeapChunkUninterruptibly(aChunk); @@ -282,6 +292,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; extractUnalignedHeapChunkUninterruptibly(uChunk); @@ -350,6 +361,7 @@ private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); @@ -362,6 +374,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); @@ -408,6 +421,7 @@ private Object copyAlignedObject(Object originalObj) { } /** Promote an AlignedHeapChunk by moving it to this space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); @@ -425,6 +439,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } /** Promote an UnalignedHeapChunk by moving it to this Space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); @@ -441,6 +456,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { @@ -455,6 +471,7 @@ private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { return chunk; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void absorb(Space src) { /* * Absorb the chunks of a source into this Space. I cannot just copy the lists, because each diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index b86261133cca..a62552c92310 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -46,10 +47,12 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Timer open() { return openAt(System.nanoTime()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Timer openAt(long nanoTime) { openNanos = nanoTime; wasOpened = true; @@ -59,10 +62,12 @@ Timer openAt(long nanoTime) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void close() { closeAt(System.nanoTime()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void closeAt(long nanoTime) { closeNanos = nanoTime; wasClosed = true; @@ -77,6 +82,7 @@ public void reset() { collectedNanos = 0L; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long getOpenedTime() { if (!wasOpened) { /* If a timer was not opened, pretend it was opened at the start of the VM. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index b6673242aec1..8a1ef2eaa98f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -145,6 +145,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 1c997255d9af..d7f337a6a0b4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -132,6 +132,7 @@ Space getSurvivorFromSpaceAt(int index) { return survivorFromSpaces[index]; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { return survivorGreyObjectsWalkers[index]; } @@ -164,6 +165,7 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void prepareForPromotion() { for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorToSpaceAt(i).isEmpty() : "SurvivorToSpace should be empty."; @@ -171,8 +173,8 @@ void prepareForPromotion() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -184,10 +186,8 @@ boolean scanGreyObjects() { return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).string("]").newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); } - trace.string("]").newline(); return true; } @@ -238,6 +238,7 @@ UnsignedWord computeSurvivorObjectBytes() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") public boolean contains(Object object) { if (!HeapImpl.usesImageHeapCardMarking()) { @@ -251,6 +252,7 @@ public boolean contains(Object object) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -266,6 +268,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -281,6 +284,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { assert originalSpace.isFromSpace(); assert originalSpace.getAge() < maxSurvivorSpaces; @@ -298,6 +302,7 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig return true; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { if (isAligned) { return alignedChunkFitsInSurvivors(); @@ -305,17 +310,20 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(size); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; if (!alignedChunkFitsInSurvivors()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index d244d47c7cb3..8c678b33e9ad 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -37,6 +37,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -86,6 +87,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -95,6 +97,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(AlignedHeader chunk) { // Completely clean the card table and the first object table as further objects may be // added later on to this chunk. @@ -110,6 +113,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(AlignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -118,6 +122,7 @@ public static void clearRememberedSet(AlignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object object, boolean verifyOnly) { Pointer objectPointer = Word.objectToUntrackedPointer(object); AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer); @@ -130,6 +135,7 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); Pointer objectsLimit = HeapChunk.getTopPointer(chunk); @@ -183,6 +189,7 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -203,6 +210,7 @@ public static boolean verify(AlignedHeader chunk) { } /** Return the index of an object within the tables of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { UnsignedWord offset = AlignedHeapChunk.getObjectOffset(chunk, objectPointer); return CardTable.memoryOffsetToIndex(offset); @@ -260,18 +268,22 @@ static UnsignedWord getCardTableLimitOffset() { return UnsignedUtils.roundUp(tableLimit, alignment); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(AlignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(Pointer chunk) { return chunk.add(getFirstObjectTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fa707695db94..d49928864d38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; @@ -83,10 +84,12 @@ final class CardTable { private CardTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate @@ -96,10 +99,12 @@ public static void setDirty(Pointer table, UnsignedWord index) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setClean(Pointer table, UnsignedWord index) { table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isDirty(Pointer table, UnsignedWord index) { int entry = readEntry(table, index); return entry == DIRTY_ENTRY; @@ -110,23 +115,28 @@ private static boolean isClean(Pointer table, UnsignedWord index) { return entry == CLEAN_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int readEntry(Pointer table, UnsignedWord index) { return table.readByte(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer cardToHeapAddress(Pointer cardTableStart, Pointer cardAddr, Pointer objectsStart) { UnsignedWord offset = cardAddr.subtract(cardTableStart).multiply(CardTable.BYTES_COVERED_BY_ENTRY); return objectsStart.add(offset); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord tableSizeForMemorySize(UnsignedWord memorySize) { return indexLimitForMemorySize(memorySize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) { UnsignedWord roundedMemory = UnsignedUtils.roundUp(memorySize, WordFactory.unsigned(BYTES_COVERED_BY_ENTRY)); return CardTable.memoryOffsetToIndex(roundedMemory); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 57d0cf1f6148..ca4bd4b7409b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -32,6 +32,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -87,33 +88,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @@ -132,6 +139,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { if (holderObject == null || object == null) { return; @@ -164,16 +172,19 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) { AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 2c5f3ff3b141..6c927f47cf0d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; @@ -163,6 +164,7 @@ final class FirstObjectTable { private FirstObjectTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initializeTable(Pointer table, UnsignedWord size) { if (SubstrateUtil.HOSTED) { // Initialize this table unconditionally as this simplifies a few things. @@ -173,12 +175,14 @@ public static void initializeTable(Pointer table, UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean doInitializeTable(Pointer table, UnsignedWord size) { UnmanagedMemoryUtil.fill(table, size, (byte) UNINITIALIZED_ENTRY); return true; } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setTableForObject(Pointer table, UnsignedWord startOffset, UnsignedWord endOffset) { assert startOffset.belowThan(endOffset); UnsignedWord startIndex = memoryOffsetToIndex(startOffset); @@ -232,6 +236,7 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un * outside the current card. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { Pointer result; Pointer firstObject = getFirstObject(tableStart, objectsStart, index); @@ -249,6 +254,7 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { UnsignedWord currentIndex = index; int currentEntry = getEntryAtIndex(tableStart, currentIndex); @@ -278,6 +284,8 @@ private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, return result; } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) { assert isMemoryOffsetEntry(entry) : "Entry out of bounds."; UnsignedWord entryOffset = WordFactory.unsigned(-entry).multiply(memoryOffsetScale()); @@ -328,69 +336,83 @@ private static UnsignedWord getTableSizeForMemoryRange(Pointer memoryStart, Poin * The multiplier from memory offsets to byte offsets into the previous card. This is the * granularity to which I can point to the start of an object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetScale() { return ConfigurationValues.getObjectLayout().getAlignment(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getEntryAtIndex(Pointer table, UnsignedWord index) { return table.readByte(indexToTableOffset(index)); } /** Set the table entry at a given index. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value) { assert isValidEntry(value) : "Invalid entry"; assert isUninitializedIndex(table, index) || getEntryAtIndex(table, index) == value : "Overwriting!"; table.writeByte(indexToTableOffset(index), (byte) value); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedIndex(Pointer table, UnsignedWord index) { int entry = getEntryAtIndex(table, index); return isUninitializedEntry(entry); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isValidEntry(int entry) { return ENTRY_MIN <= entry && entry <= ENTRY_MAX; } /** May only be used for assertions. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return entry == UNINITIALIZED_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isMemoryOffsetEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return MEMORY_OFFSET_MIN <= entry && entry <= MEMORY_OFFSET_MAX; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int biasExponent(int exponent) { assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent + EXPONENT_BIAS; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int unbiasExponent(int entry) { int exponent = entry - EXPONENT_BIAS; assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord exponentToOffset(int n) { assert 0 <= n && n <= 63 : "Exponent out of bounds."; return WordFactory.unsigned(1L << n); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToTableOffset(UnsignedWord index) { return index.multiply(ENTRY_SIZE_BYTES); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToMemoryOffset(UnsignedWord index) { return index.multiply(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetToEntry(UnsignedWord memoryOffset) { assert memoryOffset.belowThan(BYTES_COVERED_BY_ENTRY) : "Offset out of bounds."; UnsignedWord scaledOffset = memoryOffset.unsignedDivide(memoryOffsetScale()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 5f61e5c4d38c..8f7b870b487c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -34,6 +34,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -85,32 +86,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { // Nothing to do. } @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { // Nothing to do. } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return false; } @@ -127,21 +135,25 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index d6b1725a2967..d641f6569a83 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.Space; @@ -76,28 +77,35 @@ static RememberedSet get() { * Enables remembered set tracking for an aligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(AlignedHeader chunk); /** * Enables remembered set tracking for an unaligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(UnalignedHeader chunk); /** * Enables remembered set tracking for a single object in an aligned chunk. Must be called when * an object is added to the image heap or old generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForObject(AlignedHeader chunk, Object obj); /** Clears the remembered set of an aligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(AlignedHeader chunk); /** Clears the remembered set of an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(UnalignedHeader chunk); /** Checks if remembered set tracking is enabled for an object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean hasRememberedSet(UnsignedWord header); /** @@ -122,21 +130,25 @@ static RememberedSet get() { * tracking is enabled. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void dirtyCardIfNecessary(Object holderObject, Object object); /** * Walks all dirty objects in an aligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean); /** * Walks all dirty objects in an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean); /** * Walks all dirty objects in a {@link Space}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 1e993fb3f30f..c59c995c714b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; @@ -59,6 +60,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk) { // The remembered set bit in the header will be set by the code that writes the objects. } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. @@ -67,6 +69,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -75,6 +78,7 @@ public static void clearRememberedSet(UnalignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object obj, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); @@ -86,6 +90,7 @@ public static void dirtyCardForObject(Object obj, boolean verifyOnly) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer rememberedSetStart = getCardTableStart(chunk); UnsignedWord objectIndex = getObjectIndex(); @@ -132,10 +137,12 @@ static UnsignedWord getObjectIndex() { return WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java index dc4a803d0cd1..b0ed8c238fca 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java @@ -31,18 +31,14 @@ import org.graalvm.compiler.options.Option; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CEntryPointLiteral; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.IsolateListenerSupport; @@ -59,7 +55,6 @@ import com.oracle.svm.core.jfr.JfrFeature; import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler; import com.oracle.svm.core.option.HostedOptionKey; -import com.oracle.svm.core.posix.headers.Pthread; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Time; import com.oracle.svm.core.sampler.SubstrateSigprofHandler; @@ -136,38 +131,6 @@ protected void uninstallSignalHandler() { updateInterval(0); } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UnsignedWord createNativeThreadLocal() { - Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); - PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); - return key.read(); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void deleteNativeThreadLocal(UnsignedWord key) { - int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); - PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void setNativeThreadLocalValue(UnsignedWord key, IsolateThread value) { - int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); - PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected IsolateThread getNativeThreadLocalValue(UnsignedWord key) { - /* - * Although this method is not async-signal-safe in general we rely on - * implementation-specific behavior here. - */ - return (IsolateThread) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); - } - public static class Options { @Option(help = "Determines if JFR uses a signal handler for execution sampling.")// public static final HostedOptionKey SignalHandlerBasedExecutionSampler = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index 1386cf5461a3..a7a165ac0a75 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -371,6 +371,7 @@ private static long remainingNanos(long waitNanos, long startNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { PthreadVMLockSupport.checkResult(Pthread.pthread_cond_signal(getStructPointer()), "pthread_cond_signal"); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index fdc2cc341257..ed9d99e28d4e 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -39,6 +39,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; @@ -267,6 +268,39 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre return status == 0; } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return (ThreadLocalKey) key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unchecked") + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + /* + * Although this method is not async-signal-safe in general we rely on + * implementation-specific behavior here. + */ + return (T) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + @Override @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java index 55980c4a1f55..4fd44af9e3ff 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java @@ -359,6 +359,7 @@ public long blockNoTransition(long waitNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { Process.NoTransitions.WakeConditionVariable(getStructPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java index a480231284ed..85104becd0a4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java @@ -27,9 +27,9 @@ import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.RestrictHeapAccess; /** A walker over different kinds of allocated memory. */ public final class MemoryWalker { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index c6a4372a42d8..0d9e14f7d4ec 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -988,11 +988,11 @@ public int maxInvocationCount() { public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLevel, int invocationCount) { String[] args = ImageSingletons.lookup(JavaMainWrapper.JavaMainSupport.class).mainArgs; if (args != null) { - log.string("Command line: ").newline(); + log.string("Command line: "); for (String arg : args) { log.string("'").string(arg).string("' "); } - log.newline(); + log.newline().newline(); } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java index 980cd605c997..a931b4b974dc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java @@ -192,7 +192,7 @@ protected static void printSegfaultAddressInfo(Log log, long addr) { if (addr != 0) { long delta = addr - CurrentIsolate.getIsolate().rawValue(); String sign = (delta >= 0 ? "+" : "-"); - log.string("(heapBase ").string(sign).string(" ").signed(Math.abs(delta)).string(")"); + log.string(" (heapBase ").string(sign).string(" ").signed(Math.abs(delta)).string(")"); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index cb8b929c1db0..a63a9c01afc3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -43,8 +43,8 @@ import com.oracle.svm.core.JavaMemoryUtil; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; @@ -443,6 +443,7 @@ public static void setObject(NonmovableObjectArray array, int index, T va /** * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor) { if (array.isNonNull()) { return walkUnmanagedObjectArray(array, visitor, 0, lengthOf(array)); @@ -453,6 +454,7 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O /** * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor, int startIndex, int count) { if (array.isNonNull()) { assert startIndex >= 0 && count <= lengthOf(array) - startIndex; @@ -460,7 +462,7 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O assert refSize == (1 << readElementShift(array)); Pointer p = ((Pointer) array).add(readArrayBase(array)).add(startIndex * refSize); for (int i = 0; i < count; i++) { - if (!visitor.visitObjectReference(p, true, null)) { + if (!callVisitor(visitor, p)) { return false; } p = p.add(refSize); @@ -469,6 +471,11 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer p) { + return visitor.visitObjectReference(p, true, null); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void tearDown() { assert runtimeArraysInExistence.get() == 0 : "All runtime-allocated NonmovableArrays must have been freed"; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index ffdddf3856d3..1ce82946f0cd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -152,6 +152,7 @@ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); } + @Uninterruptible(reason = "Not really uninterruptible, but we are about to fail.", calleeMustBe = false) public static RuntimeException reportNoReferenceMap(Pointer sp, CodePointer ip, CodeInfo info) { Log.log().string("ip: ").hex(ip).string(" sp: ").hex(sp).string(" info:"); CodeInfoAccess.log(info, Log.log()).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 996f1089c4a0..7332fbffbef5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -153,6 +153,7 @@ public static boolean areAllObjectsOnImageHeap(CodeInfo info) { /** * Walks all strong references in a {@link CodeInfo} object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor visitor) { return NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor, CodeInfoImpl.FIRST_STRONGLY_REFERENCED_OBJFIELD, CodeInfoImpl.STRONGLY_REFERENCED_OBJFIELD_COUNT); } @@ -161,6 +162,7 @@ public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor * Walks all weak references in a {@link CodeInfo} object. */ @DuplicatedInNativeCode + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor visitor) { CodeInfoImpl impl = cast(info); boolean continueVisiting = true; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index a2bc227f563b..cdb27e1dbc56 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -201,14 +201,16 @@ private void rehashAfterUnregisterAt(int index) { // from IdentityHashMap: Knuth } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { assert VMOperation.isGCInProgress() : "otherwise, we would need to make sure that the CodeInfo is not freeded by the GC"; if (table.isNonNull()) { int length = NonmovableArrays.lengthOf(table); for (int i = 0; i < length;) { - UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); - if (info.isNonNull()) { - visitor.visitCode(CodeInfoAccess.convert(info)); + UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); + if (untetheredInfo.isNonNull()) { + CodeInfo info = CodeInfoAccess.convert(untetheredInfo); + callVisitor(visitor, info); } /* @@ -216,7 +218,7 @@ public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { * visit the now updated entry one more time. However, this could have the effect * that some entries are visited more than once. */ - if (info == NonmovableArrays.getWord(table, i)) { + if (untetheredInfo == NonmovableArrays.getWord(table, i)) { i++; } } @@ -228,23 +230,24 @@ public void walkRuntimeMethods(CodeInfoVisitor visitor) { if (table.isNonNull()) { int length = NonmovableArrays.lengthOf(table); for (int i = 0; i < length; i++) { - UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); - if (info.isNonNull()) { - Object tether = CodeInfoAccess.acquireTether(info); + UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); + if (untetheredInfo.isNonNull()) { + Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { - callVisitor(visitor, info, tether); + CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); + callVisitor(visitor, info); } finally { - CodeInfoAccess.releaseTether(info, tether); + CodeInfoAccess.releaseTether(untetheredInfo, tether); } - assert info == NonmovableArrays.getWord(table, i); + assert untetheredInfo == NonmovableArrays.getWord(table, i); } } } } - @Uninterruptible(reason = "Call the visitor, which may execute interruptible code.", calleeMustBe = false) - private static void callVisitor(CodeInfoVisitor visitor, UntetheredCodeInfo info, Object tether) { - visitor.visitCode(CodeInfoAccess.convert(info, tether)); + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static void callVisitor(CodeInfoVisitor visitor, CodeInfo info) { + visitor.visitCode(info); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index febd8215da9b..abf3f93fc044 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -75,11 +75,13 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob } /** The minimum alignment of objects (instances and arrays). */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getAlignment() { return objectAlignment; } /** Tests if the given offset or address is aligned according to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAligned(final long value) { return (value % getAlignment() == 0L); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java index a85773cdd5e0..6eba6c4dfc5f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java @@ -186,7 +186,7 @@ public static long decodeSign(long value) { } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitObjectReferenceInline(ObjectReferenceVisitor visitor, Pointer derivedRef, int innerOffset, boolean compressed, Object holderObject) { return visitor.visitObjectReferenceInline(derivedRef, innerOffset, compressed, holderObject); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index 0c8034d51148..d7a10f67ec75 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -29,6 +29,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.util.DuplicatedInNativeCode; @@ -38,6 +39,7 @@ @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { assert ReferenceMapIndex.denotesValidReferenceMap(referenceMapIndex); assert referenceMapEncoding.isNonNull(); @@ -61,7 +63,7 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra Pointer objRef = baseAddress.add(offset); for (int c = 0; c < count; c++) { - final boolean visitResult = visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); + final boolean visitResult = callVisitor(visitor, holderObject, compressed, objRef); if (!visitResult) { return false; } @@ -70,4 +72,9 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra } return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectReferenceVisitor visitor, Object holderObject, boolean compressed, Pointer objRef) { + return visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index 25df0155520a..d74bd5435bf0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -68,6 +68,7 @@ public DynamicHub dynamicHubFromObjectHeader(Word header) { return (DynamicHub) extractPotentialDynamicHubFromHeader(header).toObject(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 86fb9c99d2d3..2daabb05d6ec 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; @@ -38,6 +39,7 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } + @Uninterruptible(reason = "Not uninterruptible but it doesn't matter for the callers.", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index bd10eb301a5a..913455f96153 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -34,16 +34,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.JavaMemoryUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); @@ -55,11 +58,11 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco int gap; do { mapOffset = mapOffset.subtract(2); - gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); - nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); + gap = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); + nrefs = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); for (int i = 0; i < nrefs; i++) { - if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) { + if (!callVisitor(baseAddress, visitor, obj, isCompressed, refOffset)) { return false; } refOffset = refOffset.add(referenceSize); @@ -70,6 +73,11 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(Pointer baseAddress, ObjectReferenceVisitor visitor, Object obj, boolean isCompressed, UnsignedWord refOffset) { + return visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj); + } + /** * Implements the allocation and the copying of the reference map and data stored in the hybrid * array for {@link Object#clone()}. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java index 68605e09b182..f0c879cd8387 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java @@ -69,6 +69,7 @@ static Reference uncast(Target_java_lang_ref_Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#referent} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset))); } @@ -79,7 +80,7 @@ public static T getReferent(Reference instance) { } /** Write {@link Target_java_lang_ref_Reference#referent}. */ - @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setReferent(Reference instance, Object value) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), value); } @@ -105,6 +106,7 @@ public static void clear(Reference instance) { ObjectAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), null); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentFieldAddress(Reference instance) { return Word.objectToUntrackedPointer(instance).add(WordFactory.unsigned(Target_java_lang_ref_Reference.referentFieldOffset)); } @@ -119,6 +121,7 @@ public static Reference getNextDiscovered(Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#discovered} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getDiscoveredPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset))); } @@ -140,6 +143,7 @@ public static boolean isAnyReferenceFieldOffset(long offset) { } /** Write {@link Target_java_lang_ref_Reference#discovered}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setNextDiscovered(Reference instance, Reference newNext) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset), newNext); } @@ -251,6 +255,7 @@ public static boolean waitForReferenceProcessing() throws InterruptedException { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceClock() { return Target_java_lang_ref_SoftReference.clock; } @@ -262,6 +267,7 @@ public static void updateSoftReferenceClock() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceTimestamp(SoftReference instance) { Target_java_lang_ref_SoftReference ref = SubstrateUtil.cast(instance, Target_java_lang_ref_SoftReference.class); return ref.timestamp; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java index 4989c633cbbe..e80b89687eeb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + public class ReferenceMapIndex { /** * Marker value returned by @@ -44,6 +46,7 @@ public static boolean denotesEmptyReferenceMap(long referenceMapIndex) { return referenceMapIndex == EMPTY_REFERENCE_MAP || referenceMapIndex == NO_REFERENCE_MAP; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean denotesValidReferenceMap(long referenceMapIndex) { return referenceMapIndex != NO_REFERENCE_MAP; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index d228ddb2166e..7be91179acd7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -618,6 +618,7 @@ public DynamicHub getArrayHub() { return arrayHub; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getReferenceMapIndex() { return referenceMapIndex; } @@ -662,6 +663,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getHubType() { return hubType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index fe2a8f6c4509..989fad267fa8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; @@ -66,6 +67,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -107,6 +109,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { NonmovableArray referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding(); long referenceMapIndex = objHub.getReferenceMapIndex(); @@ -116,6 +119,7 @@ private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); @@ -124,6 +128,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!Continuation.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); @@ -132,11 +137,13 @@ private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkOther() { throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { int length = ArrayLengthNode.arrayLength(obj); int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); @@ -145,7 +152,7 @@ private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visito Pointer pos = objPointer.add(LayoutEncoding.getArrayBaseOffset(objHub.getLayoutEncoding())); Pointer end = pos.add(WordFactory.unsigned(referenceSize).multiply(length)); while (pos.belowThan(end)) { - final boolean visitResult = visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); + final boolean visitResult = callVisitor(obj, visitor, isCompressed, pos); if (!visitResult) { return false; } @@ -153,4 +160,9 @@ private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visito } return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(Object obj, ObjectReferenceVisitor visitor, boolean isCompressed, Pointer pos) { + return visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index e13f5e27379a..b1f30bc5a7d8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -329,16 +329,19 @@ public static UnsignedWord getMomentarySizeFromObject(Object obj) { return getSizeFromObject(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInGC(Object obj) { return getSizeFromObjectInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); @@ -365,11 +368,13 @@ private static boolean checkOptionalIdentityHashField(Object obj) { return oh.hasOptionalIdentityHashField(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInGC(Object obj) { return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInlineInGC(Object obj) { UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); return Word.objectToUntrackedPointer(obj).add(size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index edb8c09bb17b..143954b7ed18 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -435,6 +435,14 @@ public static long abs(long a) { } } + public static class Byte { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("cast") + public static int toUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + } + public static class Long { /** Uninterruptible version of {@link java.lang.Long#numberOfLeadingZeros(long)}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java index cb6a17539608..ce0e926ebee1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java @@ -177,6 +177,7 @@ public long blockNoTransition(long nanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { /* Nothing to do. */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java index 6a5872171303..f85d789026dd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java @@ -98,6 +98,7 @@ public void blockNoTransitionUnspecifiedOwner() { /** * Wakes up a single thread that is waiting on this condition. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { throw VMError.shouldNotReachHere("VMCondition cannot be used during native image generation"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f381b6ed1b4d..9e2be0270d71 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -103,6 +103,7 @@ public int protect(PointerBase start, UnsignedWord nbytes, EnumSet acces } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, false); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index d54b95287d57..11b30062e607 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -89,6 +89,7 @@ default UnsignedWord getGranularity() { return VirtualMemoryProvider.get().getGranularity(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment); Pointer allocateUnalignedChunk(UnsignedWord nbytes); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java index 9ca0aea7e207..2aa121f33fb7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.IsolateListenerSupport.IsolateListener; @@ -44,6 +43,8 @@ import com.oracle.svm.core.graal.nodes.WriteHeapBaseNode; import com.oracle.svm.core.jfr.SubstrateJVM; import com.oracle.svm.core.jfr.sampler.AbstractJfrExecutionSampler; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.thread.ThreadListener; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; @@ -61,7 +62,7 @@ */ public abstract class SubstrateSigprofHandler extends AbstractJfrExecutionSampler implements IsolateListener, ThreadListener { private static final CGlobalData SIGNAL_HANDLER_ISOLATE = CGlobalDataFactory.createWord(); - private UnsignedWord keyForNativeThreadLocal; + private ThreadLocalKey keyForNativeThreadLocal; @Platforms(Platform.HOSTED_ONLY.class) protected SubstrateSigprofHandler() { @@ -75,15 +76,15 @@ public static SubstrateSigprofHandler singleton() { @Override @Uninterruptible(reason = "Thread state not set up yet.") public void afterCreateIsolate(Isolate isolate) { - keyForNativeThreadLocal = createNativeThreadLocal(); + keyForNativeThreadLocal = PlatformThreads.singleton().createUnmanagedThreadLocal(); } @Override @Uninterruptible(reason = "The isolate teardown is in progress.") public void onIsolateTeardown() { - UnsignedWord oldKey = keyForNativeThreadLocal; + ThreadLocalKey oldKey = keyForNativeThreadLocal; keyForNativeThreadLocal = WordFactory.nullPointer(); - deleteNativeThreadLocal(oldKey); + PlatformThreads.singleton().deleteUnmanagedThreadLocal(oldKey); } @Override @@ -175,18 +176,6 @@ private static void uninstall(IsolateThread thread) { } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract UnsignedWord createNativeThreadLocal(); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract void deleteNativeThreadLocal(UnsignedWord key); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract void setNativeThreadLocalValue(UnsignedWord key, IsolateThread value); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract IsolateThread getNativeThreadLocalValue(UnsignedWord key); - /** * Called from the platform dependent sigprof handler to enter isolate. */ @@ -205,8 +194,8 @@ protected static boolean tryEnterIsolate() { /* We are keeping reference to isolate thread inside OS thread local area. */ if (SubstrateOptions.MultiThreaded.getValue()) { - UnsignedWord key = singleton().keyForNativeThreadLocal; - IsolateThread thread = singleton().getNativeThreadLocalValue(key); + ThreadLocalKey key = singleton().keyForNativeThreadLocal; + IsolateThread thread = PlatformThreads.singleton().getUnmanagedThreadLocalValue(key); if (thread.isNull()) { /* Thread is not yet initialized or already detached from isolate. */ return false; @@ -220,6 +209,6 @@ protected static boolean tryEnterIsolate() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void storeIsolateThreadInNativeThreadLocal(IsolateThread isolateThread) { - setNativeThreadLocalValue(keyForNativeThreadLocal, isolateThread); + PlatformThreads.singleton().setUnmanagedThreadLocalValue(keyForNativeThreadLocal, isolateThread); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 3b6076c42a21..f2cb98519e78 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -64,10 +64,13 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; +import org.graalvm.word.ComparableWord; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; +import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; @@ -559,6 +562,29 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.joinThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); + } + + @SuppressWarnings("unused") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); + } + + @SuppressWarnings("unused") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); + } + + @SuppressWarnings("unused") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void closeOSThreadHandle(OSThreadHandle threadHandle) { @@ -1189,8 +1215,19 @@ static void blockedOn(Target_sun_nio_ch_Interruptible b) { } } + @RawStructure public interface OSThreadHandle extends PointerBase { } + + @RawPointerTo(OSThreadHandle.class) + public interface OSThreadHandlePointer extends PointerBase { + void write(int index, OSThreadHandle value); + + OSThreadHandle read(int index); + } + + public interface ThreadLocalKey extends ComparableWord { + } } @TargetClass(value = ThreadPoolExecutor.class, innerClass = "Worker") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java index bb192cc5a5b2..874bd588bc1c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java @@ -67,6 +67,7 @@ public int getThreadLocalsReferenceMapIndex() { return (int) result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walk(IsolateThread isolateThread, ObjectReferenceVisitor referenceVisitor) { NonmovableArray threadRefMapEncoding = NonmovableArrays.fromImageHeap(vmThreadReferenceMapEncoding); InstanceReferenceMapDecoder.walkOffsetsFromPointer((Pointer) isolateThread, threadRefMapEncoding, vmThreadReferenceMapIndex, referenceVisitor, null); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java index bc00b091f324..bc616f0a8786 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java @@ -26,6 +26,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; @@ -48,7 +49,7 @@ public static long readU4(Pointer ptr) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord readU8(Pointer ptr) { - return ptr.readWord(0); + return WordFactory.unsigned(ptr.readLong(0)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java index c372b8e1591c..15e58be1f46f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java @@ -45,6 +45,7 @@ import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.classinitialization.EnsureClassInitializedNode; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.os.RawFileOperationSupport; @@ -73,8 +74,7 @@ private static UninterruptibleAnnotationChecker singleton() { public static void checkAfterParsing(ResolvedJavaMethod method, StructuredGraph graph) { if (Uninterruptible.Utils.isUninterruptible(method) && graph != null) { - singleton().checkNoAllocation(method, graph); - singleton().checkNoSynchronization(method, graph); + singleton().checkGraph(method, graph); } } @@ -134,12 +134,6 @@ private void checkSpecifiedOptions(HostedMethod method, Uninterruptible annotati } if (annotation.mayBeInlined()) { - if (!annotation.reason().equals(Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE) && !AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { - violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible('mayBeInlined = true') which allows the method to be inlined into interruptible code. " + - "If the method has an inherent reason for being uninterruptible, besides being called from uninterruptible code, then please remove 'mayBeInlined = true'. " + - "Otherwise, use the following reason: '" + Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE + "'"); - } - if (AnnotationAccess.isAnnotationPresent(method, NeverInline.class)) { violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with conflicting annotations: @Uninterruptible('mayBeInlined = true') and @NeverInline"); @@ -151,6 +145,14 @@ private void checkSpecifiedOptions(HostedMethod method, Uninterruptible annotati } } + if (annotation.mayBeInlined() && annotation.calleeMustBe()) { + if (!annotation.reason().equals(Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE) && !AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { + violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible('mayBeInlined = true') which allows the method to be inlined into interruptible code. " + + "If the method has an inherent reason for being uninterruptible, besides being called from uninterruptible code, then please remove 'mayBeInlined = true'. " + + "Otherwise, use the following reason: '" + Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE + "'"); + } + } + if (!annotation.mayBeInlined() && !annotation.callerMustBe() && AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible and @AlwaysInline. If the method may be inlined into interruptible code, please specify 'mayBeInlined = true'. Otherwise, specify 'callerMustBe = true'."); @@ -259,18 +261,14 @@ private void checkCallers(HostedMethod caller, Uninterruptible callerAnnotation, } } - private void checkNoAllocation(ResolvedJavaMethod method, StructuredGraph graph) { + private void checkGraph(ResolvedJavaMethod method, StructuredGraph graph) { for (Node node : graph.getNodes()) { if (isAllocationNode(node)) { violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " is not allowed to allocate."); - } - } - } - - private void checkNoSynchronization(ResolvedJavaMethod method, StructuredGraph graph) { - for (Node node : graph.getNodes()) { - if (node instanceof MonitorEnterNode) { + } else if (node instanceof MonitorEnterNode) { violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " is not allowed to use 'synchronized'."); + } else if (node instanceof EnsureClassInitializedNode) { + violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " not allowed to do class initialization."); } } }