diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index bee7b2871e1b..1f8e4b4308fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { assert VMOperation.isGCInProgress() : "use only during GC"; guaranteeSizeParametersInitialized(); @@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index bc15be90bdcf..e87e6b922d30 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static public interface AlignedHeader extends HeapChunk.Header { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize); } @@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) { } /** Allocate uninitialized memory within this AlignedHeapChunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { Pointer result = WordFactory.nullPointer(); UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -112,6 +114,15 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } + /** Retract the latest allocation. Used by parallel collector. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { + Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); + assert newTop.aboveOrEqual(getObjectsStart(that)); + HeapChunk.setTopPointer(that, newTop); + return newTop; + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static UnsignedWord getCommittedObjectMemory(AlignedHeader that) { return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset()); @@ -129,6 +140,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { } /** Return the offset of an object within the objects part of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) { Pointer objectsStart = getObjectsStart(that); return objectPointer.subtract(objectsStart); @@ -139,7 +151,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") - static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { + public static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } @@ -154,7 +166,7 @@ static MemoryWalker.HeapChunkAccess getMemoryWal } /** Methods for a {@link MemoryWalker} to access an aligned heap chunk. */ - @AutomaticallyRegisteredImageSingleton(onlyWith = UseSerialOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java index ae89d3740679..8aa186b7509c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; @@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { return WordFactory.zero(); } @@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { return 1; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 17c24b576832..02244cea6047 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -60,6 +60,7 @@ public void reset() { unalignedChunkBytes = WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getChunkBytes() { return getAlignedChunkBytes().add(getUnalignedChunkBytes()); } @@ -96,6 +97,7 @@ void noteAlignedHeapChunk() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteAlignedHeapChunk() { alignedCount--; if (parent != null) { @@ -117,10 +119,12 @@ private void noteUnaligned(UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void unnoteUnaligned(UnsignedWord size) { unalignedCount--; unalignedChunkBytes = unalignedChunkBytes.subtract(size); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 1d11025ef0f7..ba738c3df223 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; @@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x * this amount can be used for surviving objects. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnsignedWord getSurvivorSpacesCapacity(); /** The capacity of the young generation, comprising the eden and survivor spaces. */ @@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of * survivor spaces + 1}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getTenuringAge(); /** Called at the beginning of a collection, in the safepoint operation. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 128df06b6087..c006ab6faa54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) { /** Called after an object has been promoted from the young generation to the old generation. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index fc8419d5a49a..f0ec2c931709 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -64,6 +64,7 @@ import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.graal.RuntimeCompilation; import com.oracle.svm.core.heap.CodeReferenceMapDecoder; @@ -128,6 +129,8 @@ public final class GCImpl implements GC { public String getName() { if (SubstrateOptions.UseEpsilonGC.getValue()) { return "Epsilon GC"; + } else if (SubstrateOptions.UseParallelGC.getValue()) { + return "Parallel GC"; } else { return "Serial GC"; } @@ -198,6 +201,10 @@ private void collectOperation(CollectionVMOperationData data) { assert VMOperation.isGCInProgress() : "Collection should be a VMOperation."; assert getCollectionEpoch().equal(data.getRequestingEpoch()); + if (SubstrateOptions.UseParallelGC.getValue()) { + ParallelGC.singleton().initialize(); + } + timers.mutator.closeAt(data.getRequestingNanoTime()); startCollectionOrExit(); @@ -525,6 +532,7 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } @@ -1052,6 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) { try { if (isIncremental) { scanGreyObjectsLoop(); + } else if (ParallelGC.isEnabled()) { + ParallelGC.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } @@ -1072,6 +1082,7 @@ private static void scanGreyObjectsLoop() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); @@ -1105,6 +1116,7 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1217,7 +1229,8 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } - GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { + @Fold + public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java index 4e0daf2c7dc0..3e8f6a24c48a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java @@ -46,7 +46,7 @@ public class GenScavengeMemoryPoolMXBeans { @Platforms(Platform.HOSTED_ONLY.class) public static MemoryPoolMXBean[] createMemoryPoolMXBeans() { - if (SubstrateOptions.UseSerialGC.getValue()) { + if (SubstrateOptions.useMarkAndCopyGC()) { mxBeans = new AbstractMemoryPoolMXBean[]{ new EdenMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER), new SurvivorMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER), diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 35b694891c19..e487a4c7f1d2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,6 +27,8 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -66,6 +68,8 @@ public String getName() { * promotion was done by copying, or {@code null} if there was insufficient capacity in * this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); /** @@ -79,6 +83,8 @@ public String getName() { * was promoted through HeapChunk motion, or {@code null} if there was insufficient * capacity in this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 03105a9b19fd..d13e9d9e4edd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,16 +24,17 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; /** @@ -58,13 +59,21 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + return visitObjectReferenceUninterruptibly(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + return visitObjectReferenceUninterruptibly(objRef, innerOffset, compressed, holderObject); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); @@ -101,7 +110,6 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); - assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. @@ -130,16 +138,22 @@ public interface Counters extends AutoCloseable { @Override void close(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteObjRef(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNullReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteForwardedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNonHeapReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteCopiedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); void toLog(); @@ -148,13 +162,13 @@ public interface Counters extends AutoCloseable { } public static class RealCounters implements Counters { - private long objRef; - private long nullObjRef; - private long nullReferent; - private long forwardedReferent; - private long nonHeapReferent; - private long copiedReferent; - private long unmodifiedReference; + private UninterruptibleUtils.AtomicLong objRef; + private UninterruptibleUtils.AtomicLong nullObjRef; + private UninterruptibleUtils.AtomicLong nullReferent; + private UninterruptibleUtils.AtomicLong forwardedReferent; + private UninterruptibleUtils.AtomicLong nonHeapReferent; + private UninterruptibleUtils.AtomicLong copiedReferent; + private UninterruptibleUtils.AtomicLong unmodifiedReference; RealCounters() { reset(); @@ -162,13 +176,13 @@ public static class RealCounters implements Counters { @Override public void reset() { - objRef = 0L; - nullObjRef = 0L; - nullReferent = 0L; - forwardedReferent = 0L; - nonHeapReferent = 0L; - copiedReferent = 0L; - unmodifiedReference = 0L; + objRef.set(0L); + nullObjRef.set(0L); + nullReferent.set(0L); + forwardedReferent.set(0L); + nonHeapReferent.set(0L); + copiedReferent.set(0L); + unmodifiedReference.set(0L); } @Override @@ -184,46 +198,52 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { - objRef += 1L; + objRef.incrementAndGet(); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { - nullReferent += 1L; + nullReferent.incrementAndGet(); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { - forwardedReferent += 1L; + forwardedReferent.incrementAndGet(); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { - nonHeapReferent += 1L; + nonHeapReferent.incrementAndGet(); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { - copiedReferent += 1L; + copiedReferent.incrementAndGet(); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { - unmodifiedReference += 1L; + unmodifiedReference.incrementAndGet(); } @Override public void toLog() { Log log = Log.log(); log.string("[GreyToBlackObjRefVisitor.counters:"); - log.string(" objRef: ").signed(objRef); - log.string(" nullObjRef: ").signed(nullObjRef); - log.string(" nullReferent: ").signed(nullReferent); - log.string(" forwardedReferent: ").signed(forwardedReferent); - log.string(" nonHeapReferent: ").signed(nonHeapReferent); - log.string(" copiedReferent: ").signed(copiedReferent); - log.string(" unmodifiedReference: ").signed(unmodifiedReference); + log.string(" objRef: ").signed(objRef.get()); + log.string(" nullObjRef: ").signed(nullObjRef.get()); + log.string(" nullReferent: ").signed(nullReferent.get()); + log.string(" forwardedReferent: ").signed(forwardedReferent.get()); + log.string(" nonHeapReferent: ").signed(nonHeapReferent.get()); + log.string(" copiedReferent: ").signed(copiedReferent.get()); + log.string(" unmodifiedReference: ").signed(unmodifiedReference.get()); log.string("]").newline(); } } @@ -243,26 +263,32 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 9a8b48845d46..329c626fb69d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; @@ -55,7 +56,14 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectInline(Object o) { + return visitObjectUninterruptibly(o); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectUninterruptibly(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 69104f59be47..43df19d7b1fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -41,8 +41,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.struct.PinnedObjectField; @@ -177,6 +177,7 @@ public interface Header> extends HeaderPadding { void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); @@ -306,6 +307,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 4df6532b8e51..9fcc5490b669 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -35,11 +34,11 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; @@ -89,31 +88,31 @@ public UnsignedWord getBytesInUnusedChunks() { return bytesInUnusedAlignedChunks.get(); } - @AlwaysInline("Remove all logging when noopLog is returned by this method") - private static Log log() { - return Log.noopLog(); - } - private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk"); private static final OutOfMemoryError UNALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an unaligned heap chunk"); /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { - UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); - log().string("[HeapChunkProvider.produceAlignedChunk chunk size: ").unsigned(chunkSize).newline(); + return produceAlignedChunk(true); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + AlignedHeader produceAlignedChunk(boolean reportOutOfMemory) { + UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); AlignedHeader result = popUnusedAlignedChunk(); - log().string(" unused chunk: ").zhex(result).newline(); - if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ noteFirstAllocationTime(); result = (AlignedHeader) CommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); if (result.isNull()) { - throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + if (reportOutOfMemory) { + throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + } else { + throw ALIGNED_OUT_OF_MEMORY_ERROR; + } } - log().string(" new chunk: ").zhex(result).newline(); AlignedHeapChunk.initialize(result, chunkSize); } @@ -123,8 +122,6 @@ AlignedHeader produceAlignedChunk() { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" result chunk: ").zhex(result).string(" ]").newline(); return result; } @@ -200,13 +197,10 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { if (SubstrateOptions.MultiThreaded.getValue()) { VMThreads.guaranteeOwnsThreadMutex("Should hold the lock when pushing to the global list."); } - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); HeapChunk.setNext(chunk, unusedAlignedChunks.get()); unusedAlignedChunks.set(chunk); bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize()); - - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); } /** @@ -218,15 +212,13 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { * garbage collections, I avoid the ABA problem by making the kernel of this method * uninterruptible so it can not be interrupted by a safepoint. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeader popUnusedAlignedChunk() { - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); - AlignedHeader result = popUnusedAlignedChunkUninterruptibly(); if (result.isNull()) { return WordFactory.nullPointer(); } else { bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize()); - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); return result; } } @@ -268,7 +260,6 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { /** Acquire an UnalignedHeapChunk from the operating system. */ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - log().string("[HeapChunkProvider.produceUnalignedChunk objectSize: ").unsigned(objectSize).string(" chunkSize: ").zhex(chunkSize).newline(); noteFirstAllocationTime(); UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); @@ -282,8 +273,6 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" returns ").zhex(result).string(" ]").newline(); return result; } @@ -300,10 +289,10 @@ static void consumeUnalignedChunks(UnalignedHeader firstChunk) { freeUnalignedChunkList(firstChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void zap(Header chunk, WordBase value) { Pointer start = HeapChunk.getTopPointer(chunk); Pointer limit = HeapChunk.getEndPointer(chunk); - log().string(" zap chunk: ").zhex(chunk).string(" start: ").zhex(start).string(" limit: ").zhex(limit).string(" value: ").zhex(value).newline(); for (Pointer p = start; p.belowThan(limit); p = p.add(FrameAccess.wordSize())) { p.writeWord(0, value); } @@ -332,6 +321,7 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return continueVisiting; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void noteFirstAllocationTime() { if (firstAllocationTime == 0L) { firstAllocationTime = System.nanoTime(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index f41076dd6ed1..538d223704be 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -42,22 +42,22 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunk; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunkRegistry; import com.oracle.svm.core.SubstrateDiagnostics.ErrorContext; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -69,6 +69,7 @@ import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.ReferenceInternals; +import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.locks.VMCondition; @@ -201,6 +202,9 @@ boolean walkMemory(MemoryWalker.Visitor visitor) { @Override @Uninterruptible(reason = "Tear-down in progress.") public boolean tearDown() { + if (ParallelGC.isEnabled()) { + ParallelGC.singleton().tearDown(); + } youngGeneration.tearDown(); oldGeneration.tearDown(); getChunkProvider().tearDown(); @@ -901,7 +905,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseSerialOrEpsilonGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 9c2be3f9a442..e5817b56068d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -137,6 +137,7 @@ public static UnsignedWord getLargeArrayThreshold() { * Zapping */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 47126cddf9e2..693f3990ee5c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -140,7 +140,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue(); + return SubstrateOptions.useMarkAndCopyGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 1ffadd77743a..f5e16c2c8702 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -95,7 +95,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { } else { VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); numReservedBits = numMinimumReservedBits + 2; - VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(), "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); } numReservedExtraBits = numReservedBits - numAlignmentBits; @@ -209,6 +209,7 @@ public boolean hasOptionalIdentityHashField(Word header) { return header.and(IDHASH_STATE_BITS).equal(inFieldState); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setIdentityHashInField(Object o) { assert VMOperation.isGCInProgress(); VMError.guarantee(!hasFixedIdentityHashField()); @@ -340,8 +341,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition) { - ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { if (partition.isWritable()) { header |= REMEMBERED_SET_BIT.rawValue(); } @@ -363,6 +363,7 @@ public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } @@ -378,12 +379,14 @@ public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setRememberedSetBit(Object o) { UnsignedWord oldHeader = readHeaderFromObject(o); UnsignedWord newHeader = oldHeader.or(REMEMBERED_SET_BIT); writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } @@ -399,14 +402,16 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (hasShift()) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -423,24 +428,51 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); - UnsignedWord forwardHeader; + UnsignedWord forwardHeader = getForwardHeader(copy); + ObjectAccess.writeLong(original, getHubOffset(), forwardHeader.rawValue()); + assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + } + + /** + * The original header are the 8 bytes at the hub offset (regardless if compressed references + * are used or not). + */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + Object installForwardingPointerParallel(Object original, long eightHeaderBytes, Object copy) { + UnsignedWord forwardHeader = getForwardHeader(copy); + /* Try installing the new header. */ + Pointer originalPtr = Word.objectToUntrackedPointer(original); + long value = originalPtr.compareAndSwapLong(getHubOffset(), eightHeaderBytes, forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); + assert isPointerToForwardedObject(originalPtr); + if (value != eightHeaderBytes) { + return getForwardedObject(originalPtr, WordFactory.unsigned(value)); + } + return copy; + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private UnsignedWord getForwardHeader(Object copy) { + UnsignedWord result; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + UnsignedWord compressedCopy = ReferenceAccess.singleton().getCompressedRepresentation(copy); + if (hasShift()) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + result = compressedCopy.shiftLeft(32).or(WordFactory.unsigned(0x00000000e0e0e0e0L)); } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + result = compressedCopy; } } else { - forwardHeader = Word.objectToUntrackedPointer(copy); + result = Word.objectToUntrackedPointer(copy); } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + + assert getHeaderBitsFromHeader(result).equal(0); + return result.or(FORWARDED_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -460,6 +492,11 @@ static int getReferenceSize() { return ConfigurationValues.getObjectLayout().getReferenceSize(); } + @Fold + static boolean hasShift() { + return getReferenceSize() == 4; + } + @Fold static boolean hasFixedIdentityHashField() { return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 395e39ef03fe..2bd8eba27116 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -31,8 +31,8 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -73,6 +73,7 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -80,6 +81,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -158,13 +160,13 @@ UnsignedWord getChunkBytes() { return fromBytes.add(toBytes); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") - AlignedHeapChunk.AlignedHeader requestAlignedChunk() { + AlignedHeapChunk.AlignedHeader requestAlignedChunk(boolean reportOutOfMemory) { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); + AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(reportOutOfMemory); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - Log.log().string("[! OldGeneration.requestAlignedChunk: failure to allocate aligned chunk!]"); - throw VMError.shouldNotReachHere("Promotion failure"); + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } RememberedSet.get().enableRememberedSetForChunk(chunk); return chunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index f58a5b910776..a5686c28b8f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -42,7 +42,7 @@ /** Support for pinning objects to a memory address with {@link PinnedObject}. */ final class PinnedObjectImpl implements PinnedObject { - @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseSerialOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) static class PinnedObjectSupportImpl implements PinnedObjectSupport { @Override public PinnedObject create(Object object) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 429afe8fa0b9..83c2169b227c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,11 +36,13 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceInternals; import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; @@ -48,7 +50,7 @@ /** Discovers and handles {@link Reference} objects during garbage collection. */ final class ReferenceObjectProcessing { /** Head of the linked list of discovered references that need to be revisited. */ - private static Reference rememberedRefsList; + private static final AtomicReference> rememberedRefsList = new AtomicReference<>(); /** * For a {@link SoftReference}, the longest duration after its last access to keep its referent @@ -78,6 +80,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -86,6 +89,8 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header @@ -141,9 +146,11 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { // are revisited after the GC finished promoting all strongly reachable objects. // null link means undiscovered, avoid for the last node with a cyclic reference - Reference next = (rememberedRefsList != null) ? rememberedRefsList : dr; - ReferenceInternals.setNextDiscovered(dr, next); - rememberedRefsList = dr; + Reference expected; + do { + expected = rememberedRefsList.get(); + ReferenceInternals.setNextDiscovered(dr, expected != null ? expected : dr); + } while (!rememberedRefsList.compareAndSet(expected, dr)); } /** @@ -154,8 +161,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { */ static Reference processRememberedReferences() { Reference pendingHead = null; - Reference current = rememberedRefsList; - rememberedRefsList = null; + Reference current = rememberedRefsList.getAndSet(null); while (current != null) { // Get the next node (the last node has a cyclic reference to self). @@ -180,7 +186,7 @@ static Reference processRememberedReferences() { } static void afterCollection(UnsignedWord freeBytes) { - assert rememberedRefsList == null; + assert rememberedRefsList.get() == null; UnsignedWord unused = freeBytes.unsignedDivide(1024 * 1024 /* MB */); maxSoftRefAccessIntervalMs = unused.multiply(SerialGCOptions.SoftRefLRUPolicyMSPerMB.getValue()); ReferenceInternals.updateSoftReferenceClock(); @@ -217,6 +223,7 @@ private static boolean processRememberedRef(Reference dr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); UnsignedWord header = ohi.readHeaderFromPointer(referentAddr); @@ -228,6 +235,7 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index 27317b42499c..046f2d033f3b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -38,14 +38,14 @@ /** Common options that can be specified for both the serial and the epsilon GC. */ public final class SerialAndEpsilonGCOptions { - @Option(help = "The maximum heap size as percent of physical memory. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "The maximum heap size as percent of physical memory. Serial, parallel, and epsilon GC only.", type = OptionType.User) // + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial, parallel, and epsilon GC only.", type = OptionType.User) // + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "The size of an aligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(512 * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly) { + @Option(help = "The size of an aligned chunk. Serial, parallel, and epsilon GC only.", type = OptionType.Expert) // + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(512 * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -57,31 +57,32 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * This should be a fraction of the size of an aligned chunk, else large small arrays will not * fit in an aligned chunk. */ - @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. Serial, parallel, and epsilon GC only.", type = OptionType.Expert) // + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "Fill unused memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "Fill unused memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "Before use, fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "Before use, fill memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "After use, Fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "After use, Fill memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); - @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); private SerialAndEpsilonGCOptions() { } - public static void serialOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue()) { + private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial') or the epsilon garbage collector ('--gc=epsilon')."); + "The option '" + optionKey.getName() + + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index bc23ae04dc97..3c2778876cbc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -39,16 +39,16 @@ /** Options that are only valid for the serial GC (and not for the epsilon GC). */ public final class SerialGCOptions { @Option(help = "The garbage collection policy, either Adaptive (default) or BySpaceAndTime. Serial GC only.", type = OptionType.User)// - public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::serialGCOnly); + public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::markAndCopyGCOnly); @Option(help = "Percentage of total collection time that should be spent on young generation collections. Serial GC with collection policy 'BySpaceAndTime' only.", type = OptionType.User)// - public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::markAndCopyGCOnly); @Option(help = "The maximum free bytes reserved for allocations, in bytes (0 for automatic according to GC policy). Serial GC only.", type = OptionType.User)// - public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Maximum number of survivor spaces. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly) { + public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly) { @Override public Integer getValueOrDefault(UnmodifiableEconomicMap, Object> values) { Integer value = (Integer) values.get(this); @@ -64,51 +64,51 @@ public Integer getValue(OptionValues values) { }; @Option(help = "Determines if a full GC collects the young generation separately or together with the old generation. Serial GC only.", type = OptionType.Expert) // - public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Enables card marking for image heap objects, which arranges them in chunks. Automatically enabled when supported. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "This number of milliseconds multiplied by the free heap memory in MByte is the time span " + "for which a soft reference will keep its referent alive after its last access. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the shape of the heap before and after each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print summary GC information after application main method returns. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print a time stamp at each collection, if +PrintGC or +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the time for each of the phases of each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Instrument write barriers with counters. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify the remembered set if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify all object references if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify write barriers. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Trace heap chunks during collections, if +VerboseGC and +PrintHeapShape. Serial GC only.", type = OptionType.Debug) // - public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Develop demographics of the object references visited. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); private SerialGCOptions() { } - private static void serialGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue()) { - throw new InterruptImageBuilding("The option '" + optionKey.getName() + "' can only be used together with the serial garbage collector ('--gc=serial')."); + private static void markAndCopyGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useMarkAndCopyGC()) { + throw new InterruptImageBuilding("The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial') or parallel garbage collector ('--gc=parallel')."); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 06a6e8f0da16..c234f9a10de7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -43,6 +43,7 @@ import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; @@ -108,10 +109,12 @@ boolean isEdenSpace() { return age == 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isSurvivorSpace() { return age > 0 && age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -121,14 +124,17 @@ public boolean isOldSpace() { return age == (HeapParameters.getMaxSurvivorSpaces() + 1); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getAge() { return age; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getNextAgeForPromotion() { return age + 1; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isFromSpace() { return isFromSpace; } @@ -167,7 +173,11 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemory(UnsignedWord objectSize) { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + return allocateMemoryParallel(objectSize); + } Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); @@ -181,14 +191,60 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return allocateInNewChunk(objectSize); } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer allocateMemoryParallel(UnsignedWord objectSize) { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the thread local allocation chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + } + if (result.isNonNull()) { + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ + return allocateInNewChunkParallel(oldChunk, objectSize); + } + + /** + * Retract the latest allocation. Used by parallel collector. + */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer retractAllocation(UnsignedWord objectSize) { + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); + assert oldChunk.isNonNull(); + return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(true); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk; + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + try { + ParallelGC.singleton().pushAllocChunk(oldChunk); + newChunk = requestAlignedHeapChunk(false); + } finally { + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); + } + if (newChunk.isNonNull()) { + ParallelGC.singleton().setAllocationChunk(newChunk); + return AlignedHeapChunk.allocateMemory(newChunk, objectSize); + } + return WordFactory.nullPointer(); + } + public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); @@ -200,23 +256,40 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { accounting.reset(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Must not interact with garbage collections.") void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { + appendAlignedHeapChunk(aChunk, null); + } + + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion.", true); } - appendAlignedHeapChunkUninterruptibly(aChunk); - accounting.noteAlignedHeapChunk(); - } + HeapChunk.setSpace(aChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Should only be called by the collector."; + AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); + AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstAlignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastAlignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteAlignedHeapChunk(); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { AlignedHeapChunk.AlignedHeader oldLast = getLastAlignedHeapChunk(); - HeapChunk.setSpace(aChunk, this); HeapChunk.setPrevious(aChunk, oldLast); HeapChunk.setNext(aChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -226,50 +299,43 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade if (getFirstAlignedHeapChunk().isNull()) { setFirstAlignedHeapChunk(aChunk); } - } - - void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { - assert VMOperation.isGCInProgress() : "Should only be called by the collector."; - extractAlignedHeapChunkUninterruptibly(aChunk); - accounting.unnoteAlignedHeapChunk(); + accounting.noteAlignedHeapChunk(); } @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { - AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); - AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstAlignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastAlignedHeapChunk(chunkPrev); - } - HeapChunk.setNext(aChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(aChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(aChunk, null); + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + appendUnalignedHeapChunk(uChunk, null); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion.", true); } - appendUnalignedHeapChunkUninterruptibly(uChunk); - accounting.noteUnalignedHeapChunk(uChunk); - } + HeapChunk.setSpace(uChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; + UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); + UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstUnalignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastUnalignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteUnalignedHeapChunk(uChunk); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { UnalignedHeapChunk.UnalignedHeader oldLast = getLastUnalignedHeapChunk(); - HeapChunk.setSpace(uChunk, this); HeapChunk.setPrevious(uChunk, oldLast); HeapChunk.setNext(uChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -279,32 +345,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne if (getFirstUnalignedHeapChunk().isNull()) { setFirstUnalignedHeapChunk(uChunk); } - } - - void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { - assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; - extractUnalignedHeapChunkUninterruptibly(uChunk); - accounting.unnoteUnalignedHeapChunk(uChunk); - } - - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { - UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); - UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstUnalignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastUnalignedHeapChunk(chunkPrev); - } - /* Reset the fields that the result chunk keeps for Space. */ - HeapChunk.setNext(uChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(uChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(uChunk, null); + accounting.noteUnalignedHeapChunk(uChunk); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -349,10 +390,15 @@ private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + return copyAlignedObjectParallel(original); + } + Object copy = copyAlignedObject(original); if (copy != null) { ObjectHeaderImpl.getObjectHeaderImpl().installForwardingPointer(original, copy); @@ -361,6 +407,80 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + Object copyAlignedObjectParallel(Object original) { + assert VMOperation.isGCInProgress(); + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); + assert ObjectHeaderImpl.isAlignedObject(original); + + /* + * Always read 8 bytes at the hub offset so that we can install the forwarding header with + * cmpxchng. + */ + Pointer originalMemory = Word.objectToUntrackedPointer(original); + int hubOffset = ObjectHeaderImpl.getHubOffset(); + long eightHeaderBytes = originalMemory.readLong(hubOffset); + Word originalHeader = ObjectHeaderImpl.hasShift() ? WordFactory.unsigned(eightHeaderBytes & 0xFFFFFFFFL) : WordFactory.unsigned(eightHeaderBytes); + ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); + if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { + return ohi.getForwardedObject(originalMemory, originalHeader); + } + + // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. + // If another thread copies the object first, we retract the allocation later. + UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, false); + UnsignedWord copySize = originalSize; + boolean addIdentityHashField = false; + if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(originalHeader))) { + addIdentityHashField = true; + copySize = LayoutEncoding.getSizeFromHeader(original, originalHeader, true); + } + } + + assert copySize.aboveThan(0); + Pointer copyMemory = allocateMemoryParallel(copySize); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + return null; + } + + // Install forwarding pointer into the original header + Object copy = copyMemory.toObject(); + Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); + if (forward == copy) { + // We have won the race, now we must copy the object bits. First install the original + // header + copyMemory.writeLong(hubOffset, eightHeaderBytes); + // Copy the rest of original object + if (hubOffset > 0) { + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); + } + int offset = hubOffset + Long.BYTES; + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); + + if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); + offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); + } + + if (isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + return copy; + } else { + // Retract speculatively allocated memory + retractAllocation(originalSize); + return forward; + } + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); @@ -407,11 +527,23 @@ private Object copyAlignedObject(Object originalObj) { } /** Promote an AlignedHeapChunk by moving it to this space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + } + try { + appendAlignedHeapChunk(chunk, originalSpace); + } finally { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); + } + } + } if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { @@ -424,11 +556,23 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } /** Promote an UnalignedHeapChunk by moving it to this Space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + } + try { + appendUnalignedHeapChunk(chunk, originalSpace); + } finally { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); + } + } + } if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { @@ -440,13 +584,14 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } } - private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk(boolean reportOutOfMemory) { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { assert isSurvivorSpace(); chunk = HeapImpl.getHeapImpl().getYoungGeneration().requestAlignedSurvivorChunk(); } else { - chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(); + chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(reportOutOfMemory); } if (chunk.isNonNull()) { appendAlignedHeapChunk(chunk); @@ -462,15 +607,13 @@ void absorb(Space src) { AlignedHeapChunk.AlignedHeader aChunk = src.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { AlignedHeapChunk.AlignedHeader next = HeapChunk.getNext(aChunk); - src.extractAlignedHeapChunk(aChunk); - appendAlignedHeapChunk(aChunk); + appendAlignedHeapChunk(aChunk, src); aChunk = next; } UnalignedHeapChunk.UnalignedHeader uChunk = src.getFirstUnalignedHeapChunk(); while (uChunk.isNonNull()) { UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); - src.extractUnalignedHeapChunk(uChunk); - appendUnalignedHeapChunk(uChunk); + appendUnalignedHeapChunk(uChunk, src); uChunk = next; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index b6673242aec1..1117387771ca 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -145,6 +145,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } @@ -164,7 +165,7 @@ public static MemoryWalker.HeapChunkAccess g return ImageSingletons.lookup(UnalignedHeapChunk.MemoryWalkerAccessImpl.class); } - @AutomaticallyRegisteredImageSingleton(onlyWith = UseSerialOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java similarity index 90% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java index 245874b4f0ef..7ab0a5f6d284 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java @@ -32,9 +32,9 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -public class UseSerialOrEpsilonGC implements BooleanSupplier { +public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index f25fbcf3fcd6..6a73b2e98912 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -29,8 +29,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.heap.ObjectVisitor; @@ -237,6 +237,7 @@ UnsignedWord computeSurvivorObjectBytes() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") public boolean contains(Object object) { if (!HeapImpl.usesImageHeapCardMarking()) { @@ -250,6 +251,7 @@ public boolean contains(Object object) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -265,6 +267,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -304,17 +307,20 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(size); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; if (!alignedChunkFitsInSurvivors()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 247da5e8440e..56a2b518cb2c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -204,7 +204,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useMarkAndCopyGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index 8216f42ead84..07d5f45c3d98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; +import com.oracle.svm.core.genscavenge.UseMarkAndCopyOrEpsilonGC; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -71,7 +72,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return new com.oracle.svm.core.genscavenge.UseSerialOrEpsilonGC().getAsBoolean(); + return new UseMarkAndCopyOrEpsilonGC().getAsBoolean(); } @Override @@ -160,7 +161,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.UseSerialGC.getValue()) { + if (SubstrateOptions.useMarkAndCopyGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java new file mode 100644 index 000000000000..7a99261a7afc --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.genscavenge.parallel; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.VMError; + +/** + * Synchronized buffer that stores "grey" heap chunks to be scanned. + */ +public class ChunkBuffer { + private static final int INITIAL_SIZE = 1024 * wordSize(); + + private Pointer buffer; + private int size = INITIAL_SIZE; + private int top; + + @Fold + static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + + @Platforms(Platform.HOSTED_ONLY.class) + ChunkBuffer() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void push(Pointer ptr) { + assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); + if (top >= size) { + size *= 2; + assert top < size; + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); + } + buffer.writeWord(top, ptr); + top += wordSize(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + Pointer pop() { + assert ParallelGC.isInParallelPhase(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (top > 0) { + top -= wordSize(); + return buffer.readWord(top); + } else { + return WordFactory.nullPointer(); + } + } finally { + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); + } + } + + boolean isEmpty() { + assert !ParallelGC.isInParallelPhase(); + return top == 0; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void release() { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java new file mode 100644 index 000000000000..2b63dd122cd0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.genscavenge.parallel; + +import java.util.function.BooleanSupplier; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.IsolateArgumentParser; +import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.genscavenge.AlignedHeapChunk; +import com.oracle.svm.core.genscavenge.GCImpl; +import com.oracle.svm.core.genscavenge.HeapChunk; +import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; +import com.oracle.svm.core.heap.OutOfMemoryUtil; +import com.oracle.svm.core.jdk.Jvm; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; +import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; +import com.oracle.svm.core.util.VMError; + +/** + * A garbage collector that tries to shorten GC pause by executing work on multiple "worker threads". + * Currently the only phase supported is scanning grey objects. Number of worker threads can be set with + * a runtime option. + * + * Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the synchronized + * {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for references to live + * objects to be promoted. For each object being promoted, they speculatively allocate memory for its copy + * in the to-space, then compete to install forwarding pointer in the original object. The winning thread + * proceeds to copy object data, losing threads retract the speculatively allocated memory. + * + * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As allocation + * chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push cycle continues until + * the chunk buffer becomes empty. At this point, worker threads are parked and the GC routine continues on + * the main GC thread. + */ +public class ParallelGC { + + public static final int UNALIGNED_BIT = 0x01; + + private final CEntryPointLiteral gcWorkerRunFunc; + + public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); + private final VMCondition seqPhase = new VMCondition(mutex); + private final VMCondition parPhase = new VMCondition(mutex); + private final ChunkBuffer buffer = new ChunkBuffer(); + + private boolean initialized; + private OSThreadHandlePointer workerThreads; + private GCWorkerThreadState workerStates; + private int numWorkerThreads; + private int busyWorkerThreads; + private ThreadLocalKey workerStateTL; + private volatile boolean inParallelPhase; + private volatile boolean shutdown; + private volatile Throwable error; + + @Platforms(Platform.HOSTED_ONLY.class) + public ParallelGC() { + gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); + } + + @Fold + public static ParallelGC singleton() { + return ImageSingletons.lookup(ParallelGC.class); + } + + @Fold + public static boolean isEnabled() { + return SubstrateOptions.UseParallelGC.getValue(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean isInParallelPhase() { + return singleton().inParallelPhase; + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + public AlignedHeapChunk.AlignedHeader getAllocationChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + return state.getAllocChunk(); + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert chunk.isNonNull(); + GCWorkerThreadState state = getWorkerThreadState(); + state.setAllocChunk(chunk); + state.setAllocChunkScanOffset(AlignedHeapChunk.getObjectsStartOffset()); + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + public void push(Pointer ptr) { + assert ptr.isNonNull(); + if (buffer != null) { + buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); + } + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + GCWorkerThreadState state = getWorkerThreadState(); + if (chunk.notEqual(state.getScannedChunk())) { + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); + assert scanOffset.aboveThan(0); + if (chunk.getTopOffset().aboveThan(scanOffset)) { + push(HeapChunk.asPointer(chunk).add(scanOffset)); + } + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private GCWorkerThreadState getWorkerThreadState() { + if (CurrentIsolate.getCurrentThread().isNull()) { + return PlatformThreads.singleton().getUnmanagedThreadLocalValue(workerStateTL); + } + return workerStates.addressOf(numWorkerThreads); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { + if (initialized) { + return; + } + + initialized = true; + buffer.initialize(); + + workerStateTL = PlatformThreads.singleton().createUnmanagedThreadLocal(); + + numWorkerThreads = getWorkerCount(); + busyWorkerThreads = numWorkerThreads; + + /* Allocate one struct per worker thread and one struct for the main GC thread. */ + int numWorkerStates = numWorkerThreads + 1; + workerStates = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(SizeOf.unsigned(GCWorkerThreadState.class).multiply(numWorkerStates)); + VMError.guarantee(workerStates.isNonNull()); + for (int i = 0; i < numWorkerStates; i++) { + workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); + } + + workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); + VMError.guarantee(workerThreads.isNonNull()); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); + workerThreads.write(i, thread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static int getWorkerCount() { + int index = IsolateArgumentParser.getOptionIndex(SubstrateOptions.ParallelGCThreads); + int setting = IsolateArgumentParser.getIntOptionValue(index); + int count = setting > 0 ? setting : getDefaultWorkerCount(); + return count; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static int getDefaultWorkerCount() { + // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() + int cpus = Jvm.JVM_ActiveProcessorCount(); + return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; + } + + @Uninterruptible(reason = "Heap base is not set up yet.") + @CEntryPoint(include = UseParallelGC.class, publishAs = CEntryPoint.Publish.NotPublished) + @CEntryPointOptions(prologue = GCWorkerThreadPrologue.class, epilogue = CEntryPointOptions.NoEpilogue.class) + private static void gcWorkerRun(GCWorkerThreadState state) { + try { + ParallelGC.singleton().work(state); + } catch (Throwable e) { + throw VMError.shouldNotReachHere(e); + } + } + + @NeverInline("Prevent reads from floating up.") + @Uninterruptible(reason = "Called from a GC worker thread.") + private void work(GCWorkerThreadState state) { + PlatformThreads.singleton().setUnmanagedThreadLocalValue(workerStateTL, state); + + while (true) { + Pointer ptr; + while (!inParallelPhase || error != null || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (--busyWorkerThreads == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + parPhase.blockNoTransitionUnspecifiedOwner(); + if (shutdown) { + return; + } + ++busyWorkerThreads; + } finally { + mutex.unlockNoTransitionUnspecifiedOwner(); + } + } + + try { + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable e) { + error = e; + shutdown = true; + } + } + } + + /** + * Start parallel phase and wait until all chunks have been processed. + */ + public void waitForIdle() { + GCWorkerThreadState state = getWorkerThreadState(); + assert state.getAllocChunk().isNonNull(); + push(HeapChunk.asPointer(state.getAllocChunk())); + + mutex.lock(); + try { + while (busyWorkerThreads > 0) { // wait for worker threads to become ready + debugLog().string("PP wait for workers\n"); + seqPhase.block(); + } + + debugLog().string("PP start workers\n"); + inParallelPhase = true; + parPhase.broadcast(); // let worker threads run + + while (inParallelPhase) { + debugLog().string("PP wait\n"); + seqPhase.block(); // wait for them to become idle + } + } finally { + mutex.unlock(); + } + + haltOnError(); + + assert buffer.isEmpty(); + // Reset thread local allocation chunks. + state.setAllocChunk(WordFactory.nullPointer()); + for (int i = 0; i < numWorkerThreads; i++) { + workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private static void scanChunk(Pointer ptr) { + if (ptr.isNonNull()) { + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + } else { + Pointer start = ptr; + AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); + if (chunk.equal(ptr)) { + start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); + } + HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + } + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private void scanAllocChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + if (allocChunkNeedsScanning(state)) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); + assert scanOffset.aboveThan(0); + Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); + state.setScannedChunk(allocChunk); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + state.setScannedChunk(WordFactory.nullPointer()); + if (state.getAllocChunk().equal(allocChunk)) { + // remember top offset so that we don't scan the same objects again + state.setAllocChunkScanOffset(allocChunk.getTopOffset()); + } + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); + } + + /** + * Make sure errors are thrown on main GC thread. + */ + private void haltOnError() { + if (error instanceof OutOfMemoryError oome) { + throw OutOfMemoryUtil.reportOutOfMemoryError(oome); + } else if (error instanceof Error) { + throw VMError.shouldNotReachHere(error); + } + } + + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + if (initialized) { + initialized = false; + + buffer.release(); + + // signal the worker threads so that they can shut down + shutdown = true; + parPhase.broadcast(); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = workerThreads.read(i); + PlatformThreads.singleton().joinThreadUnmanaged(thread, WordFactory.nullPointer()); + } + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); + workerThreads = WordFactory.nullPointer(); + + PlatformThreads.singleton().deleteUnmanagedThreadLocal(workerStateTL); + workerStateTL = WordFactory.nullPointer(); + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + static Log debugLog() { + return Log.noopLog(); + } + + @RawStructure + private interface GCWorkerThreadState extends PointerBase { + @RawField + Isolate getIsolate(); + + @RawField + void setIsolate(Isolate value); + + @RawField + AlignedHeapChunk.AlignedHeader getAllocChunk(); + + @RawField + void setAllocChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + AlignedHeapChunk.AlignedHeader getScannedChunk(); + + @RawField + void setScannedChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + UnsignedWord getAllocChunkScanOffset(); + + @RawField + void setAllocChunkScanOffset(UnsignedWord value); + + GCWorkerThreadState addressOf(int index); + } + + private static class GCWorkerThreadPrologue implements CEntryPointOptions.Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(GCWorkerThreadState state) { + CEntryPointSnippets.setHeapBase(state.getIsolate()); + WriteCurrentVMThreadNode.writeCurrentVMThread(WordFactory.nullPointer()); + } + } + + private static class UseParallelGC implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return SubstrateOptions.UseParallelGC.getValue(); + } + } +} + +@Platforms(Platform.HOSTED_ONLY.class) +@AutomaticallyRegisteredFeature() +@SuppressWarnings("unused") +class ParallelGCFeature implements InternalFeature { + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return ParallelGC.isEnabled(); + } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(ParallelGC.class, new ParallelGC()); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index d244d47c7cb3..f9a39b6dd305 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -37,6 +37,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -86,6 +87,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -95,6 +97,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(AlignedHeader chunk) { // Completely clean the card table and the first object table as further objects may be // added later on to this chunk. @@ -110,6 +113,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(AlignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -118,6 +122,7 @@ public static void clearRememberedSet(AlignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object object, boolean verifyOnly) { Pointer objectPointer = Word.objectToUntrackedPointer(object); AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer); @@ -203,6 +208,7 @@ public static boolean verify(AlignedHeader chunk) { } /** Return the index of an object within the tables of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { UnsignedWord offset = AlignedHeapChunk.getObjectOffset(chunk, objectPointer); return CardTable.memoryOffsetToIndex(offset); @@ -260,18 +266,22 @@ static UnsignedWord getCardTableLimitOffset() { return UnsignedUtils.roundUp(tableLimit, alignment); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(AlignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(Pointer chunk) { return chunk.add(getFirstObjectTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fa707695db94..f5421338675e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; @@ -83,10 +84,12 @@ final class CardTable { private CardTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate @@ -100,6 +103,7 @@ public static void setClean(Pointer table, UnsignedWord index) { table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isDirty(Pointer table, UnsignedWord index) { int entry = readEntry(table, index); return entry == DIRTY_ENTRY; @@ -110,10 +114,12 @@ private static boolean isClean(Pointer table, UnsignedWord index) { return entry == CLEAN_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int readEntry(Pointer table, UnsignedWord index) { return table.readByte(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 40ec1153d282..6b5e3c3b1100 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -32,6 +32,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -86,33 +87,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @@ -131,6 +138,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { if (holderObject == null || object == null) { return; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 2c5f3ff3b141..6a5274856fb3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; @@ -163,6 +164,7 @@ final class FirstObjectTable { private FirstObjectTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initializeTable(Pointer table, UnsignedWord size) { if (SubstrateUtil.HOSTED) { // Initialize this table unconditionally as this simplifies a few things. @@ -173,12 +175,14 @@ public static void initializeTable(Pointer table, UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean doInitializeTable(Pointer table, UnsignedWord size) { UnmanagedMemoryUtil.fill(table, size, (byte) UNINITIALIZED_ENTRY); return true; } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setTableForObject(Pointer table, UnsignedWord startOffset, UnsignedWord endOffset) { assert startOffset.belowThan(endOffset); UnsignedWord startIndex = memoryOffsetToIndex(startOffset); @@ -328,31 +332,37 @@ private static UnsignedWord getTableSizeForMemoryRange(Pointer memoryStart, Poin * The multiplier from memory offsets to byte offsets into the previous card. This is the * granularity to which I can point to the start of an object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetScale() { return ConfigurationValues.getObjectLayout().getAlignment(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getEntryAtIndex(Pointer table, UnsignedWord index) { return table.readByte(indexToTableOffset(index)); } /** Set the table entry at a given index. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value) { assert isValidEntry(value) : "Invalid entry"; assert isUninitializedIndex(table, index) || getEntryAtIndex(table, index) == value : "Overwriting!"; table.writeByte(indexToTableOffset(index), (byte) value); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedIndex(Pointer table, UnsignedWord index) { int entry = getEntryAtIndex(table, index); return isUninitializedEntry(entry); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isValidEntry(int entry) { return ENTRY_MIN <= entry && entry <= ENTRY_MAX; } /** May only be used for assertions. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return entry == UNINITIALIZED_ENTRY; @@ -363,6 +373,7 @@ private static boolean isMemoryOffsetEntry(int entry) { return MEMORY_OFFSET_MIN <= entry && entry <= MEMORY_OFFSET_MAX; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int biasExponent(int exponent) { assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent + EXPONENT_BIAS; @@ -379,18 +390,22 @@ private static UnsignedWord exponentToOffset(int n) { return WordFactory.unsigned(1L << n); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToTableOffset(UnsignedWord index) { return index.multiply(ENTRY_SIZE_BYTES); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToMemoryOffset(UnsignedWord index) { return index.multiply(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetToEntry(UnsignedWord memoryOffset) { assert memoryOffset.belowThan(BYTES_COVERED_BY_ENTRY) : "Offset out of bounds."; UnsignedWord scaledOffset = memoryOffset.unsignedDivide(memoryOffsetScale()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 8fec4384578c..8cbf01310aaf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -34,6 +34,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -85,32 +86,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { // Nothing to do. } @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { // Nothing to do. } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return false; } @@ -127,6 +135,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { // Nothing to do. } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index d6b1725a2967..f2d4e74e00fa 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.Space; @@ -76,28 +77,34 @@ static RememberedSet get() { * Enables remembered set tracking for an aligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(AlignedHeader chunk); /** * Enables remembered set tracking for an unaligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(UnalignedHeader chunk); /** * Enables remembered set tracking for a single object in an aligned chunk. Must be called when * an object is added to the image heap or old generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForObject(AlignedHeader chunk, Object obj); /** Clears the remembered set of an aligned chunk. */ void clearRememberedSet(AlignedHeader chunk); /** Clears the remembered set of an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(UnalignedHeader chunk); /** Checks if remembered set tracking is enabled for an object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean hasRememberedSet(UnsignedWord header); /** @@ -122,6 +129,7 @@ static RememberedSet get() { * tracking is enabled. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void dirtyCardIfNecessary(Object holderObject, Object object); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 1e993fb3f30f..1ee648709ae3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; @@ -59,6 +60,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk) { // The remembered set bit in the header will be set by the code that writes the objects. } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. @@ -67,6 +69,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -75,6 +78,7 @@ public static void clearRememberedSet(UnalignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object obj, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); @@ -132,10 +136,12 @@ static UnsignedWord getObjectIndex() { return WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index 1386cf5461a3..a7a165ac0a75 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -371,6 +371,7 @@ private static long remainingNanos(long waitNanos, long startNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { PthreadVMLockSupport.checkResult(Pthread.pthread_cond_signal(getStructPointer()), "pthread_cond_signal"); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index fdc2cc341257..fb897ea2bdc9 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -39,6 +39,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; @@ -267,6 +268,35 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre return status == 0; } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return (ThreadLocalKey) key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unchecked") + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + return (T) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + @Override @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java index 55980c4a1f55..4fd44af9e3ff 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java @@ -359,6 +359,7 @@ public long blockNoTransition(long waitNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { Process.NoTransitions.WakeConditionVariable(getStructPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 73d52698a480..9e0bbf3e0905 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -55,7 +55,7 @@ @AutomaticallyRegisteredImageSingleton public class IsolateArgumentParser { private static final RuntimeOptionKey[] OPTIONS = {SubstrateGCOptions.MinHeapSize, SubstrateGCOptions.MaxHeapSize, SubstrateGCOptions.MaxNewSize, - SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData}; + SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData, SubstrateOptions.ParallelGCThreads}; private static final int OPTION_COUNT = OPTIONS.length; private static final CGlobalData OPTION_NAMES = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNames); private static final CGlobalData OPTION_NAME_POSITIONS = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNamePosition); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 1ae2dcd9aa7e..f13ad948edd4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -65,6 +65,7 @@ import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -295,6 +296,7 @@ public String helpText() { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { + SubstrateOptions.UseParallelGC.update(values, false); SubstrateOptions.UseEpsilonGC.update(values, false); } } @@ -307,10 +309,43 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseParallelGC.update(values, false); } } }; + @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// + @Option(help = "Use a parallel GC")// + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::requireMultiThreading) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + if (newValue) { + SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseEpsilonGC.update(values, false); + } + } + }; + + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// + public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); + + private static void requireMultiThreading(OptionKey optionKey) { + if (!SubstrateOptions.MultiThreaded.getValue()) { + throw new InterruptImageBuilding( + "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); + } + } + + @Fold + public static boolean useMarkAndCopyGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue(); + } + + @Fold + public static boolean useMarkAndCopyOrEpsilonGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); + } + @Option(help = "The size of each thread stack at run-time, in bytes.", type = OptionType.User)// public static final RuntimeOptionKey StackSize = new RuntimeOptionKey<>(0L); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index bde7d510f25d..00672fbea363 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -701,6 +701,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { int derivedOffset = NumUtil.safeToInt(objRef.rawValue()); result.markReferenceAtOffset(derivedOffset, derivedOffset - innerOffset, compressed); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index febd8215da9b..abf3f93fc044 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -75,11 +75,13 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob } /** The minimum alignment of objects (instances and arrays). */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getAlignment() { return objectAlignment; } /** Tests if the given offset or address is aligned according to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAligned(final long value) { return (value % getAlignment() == 0L); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index c203d92c0fe2..af810ff48722 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -29,15 +29,17 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.util.DuplicatedInNativeCode; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.NonmovableByteArrayReader; import com.oracle.svm.core.util.TypedMemoryReader; @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { assert ReferenceMapIndex.denotesValidReferenceMap(referenceMapIndex); assert referenceMapEncoding.isNonNull(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index 5e83ff7a0932..4c3f686e0351 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -63,6 +63,7 @@ protected ObjectHeader() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public abstract DynamicHub dynamicHubFromObjectHeader(Word header); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java index f51f41d7a382..c424075096b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; @@ -44,12 +45,17 @@ public interface ObjectReferenceVisitor { boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject); /** + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + * * @param innerOffset If the reference is a {@linkplain CodeReferenceMapDecoder derived * reference}, a positive integer that must be subtracted from the address to which * the object reference points in order to get the start of the referenced object. */ @AlwaysInline("GC performance") @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { VMError.guarantee(innerOffset == 0, "visitor does not support derived references"); return visitObjectReference(objRef, compressed, holderObject); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java index ac706669506e..816e276b6d86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + /** * Supply a closure to be applied to Objects. * @@ -38,8 +40,15 @@ public interface ObjectVisitor { @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") boolean visitObject(Object o); - /** Like visitObject(Object), but inlined for performance. */ + /** + * Like visitObject(Object), but inlined for performance. + * + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectInline(Object o) { return visitObject(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 86fb9c99d2d3..8ca6e8f525f1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; @@ -38,6 +39,7 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } + @Uninterruptible(reason = "Called from uninterruptible code.", calleeMustBe = false, mayBeInlined = true) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index bd10eb301a5a..9a827a3b9c3b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -34,16 +34,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.JavaMemoryUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); @@ -55,8 +58,8 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco int gap; do { mapOffset = mapOffset.subtract(2); - gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); - nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); + gap = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); + nrefs = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); for (int i = 0; i < nrefs; i++) { if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java index 68605e09b182..f0c879cd8387 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java @@ -69,6 +69,7 @@ static Reference uncast(Target_java_lang_ref_Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#referent} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset))); } @@ -79,7 +80,7 @@ public static T getReferent(Reference instance) { } /** Write {@link Target_java_lang_ref_Reference#referent}. */ - @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setReferent(Reference instance, Object value) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), value); } @@ -105,6 +106,7 @@ public static void clear(Reference instance) { ObjectAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), null); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentFieldAddress(Reference instance) { return Word.objectToUntrackedPointer(instance).add(WordFactory.unsigned(Target_java_lang_ref_Reference.referentFieldOffset)); } @@ -119,6 +121,7 @@ public static Reference getNextDiscovered(Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#discovered} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getDiscoveredPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset))); } @@ -140,6 +143,7 @@ public static boolean isAnyReferenceFieldOffset(long offset) { } /** Write {@link Target_java_lang_ref_Reference#discovered}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setNextDiscovered(Reference instance, Reference newNext) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset), newNext); } @@ -251,6 +255,7 @@ public static boolean waitForReferenceProcessing() throws InterruptedException { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceClock() { return Target_java_lang_ref_SoftReference.clock; } @@ -262,6 +267,7 @@ public static void updateSoftReferenceClock() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceTimestamp(SoftReference instance) { Target_java_lang_ref_SoftReference ref = SubstrateUtil.cast(instance, Target_java_lang_ref_SoftReference.class); return ref.timestamp; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java index 4989c633cbbe..e80b89687eeb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + public class ReferenceMapIndex { /** * Marker value returned by @@ -44,6 +46,7 @@ public static boolean denotesEmptyReferenceMap(long referenceMapIndex) { return referenceMapIndex == EMPTY_REFERENCE_MAP || referenceMapIndex == NO_REFERENCE_MAP; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean denotesValidReferenceMap(long referenceMapIndex) { return referenceMapIndex != NO_REFERENCE_MAP; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index ae43c62f9f23..fe7d5fae0eca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -101,7 +101,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotSerialNotEpsilonGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotMarkAndCopyOrEpsilonGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -235,9 +235,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotSerialNotEpsilonGC implements BooleanSupplier { +class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue(); + return !SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index bd4ddb0dcfe1..21ca64f4dcdb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -634,6 +634,7 @@ public DynamicHub getArrayHub() { return arrayHub; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getReferenceMapIndex() { return referenceMapIndex; } @@ -678,6 +679,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getHubType() { return hubType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index fe2a8f6c4509..0aeff0c573fc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; @@ -66,6 +67,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -107,6 +109,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { NonmovableArray referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding(); long referenceMapIndex = objHub.getReferenceMapIndex(); @@ -116,6 +119,7 @@ private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); @@ -124,6 +128,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!Continuation.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); @@ -132,11 +137,13 @@ private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkOther() { throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { int length = ArrayLengthNode.arrayLength(obj); int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 1e63276c3a3c..c9dc2160814d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -329,27 +329,47 @@ public static UnsignedWord getMomentarySizeFromObject(Object obj) { return getSizeFromObject(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInGC(Object obj) { return getSizeFromObjectInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); return getSizeFromObjectInline(obj, withOptionalIdHashField); } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getSizeFromHeader(Object obj, Word header, boolean addOptionalIdHashField) { + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + DynamicHub hub = oh.dynamicHubFromObjectHeader(header); + int encoding = hub.getLayoutEncoding(); + boolean withOptionalIdHashField = addOptionalIdHashField || + (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); + return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); + } + @AlwaysInline("Actual inlining decided by callers.") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOptionalIdHashField) { DynamicHub hub = KnownIntrinsics.readHub(obj); int encoding = hub.getLayoutEncoding(); + return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static UnsignedWord getSizeFromEncoding(Object obj, DynamicHub hub, int encoding, boolean withOptionalIdHashField) { if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj), withOptionalIdHashField); } else { @@ -365,11 +385,13 @@ private static boolean checkOptionalIdentityHashField(Object obj) { return oh.hasOptionalIdentityHashField(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInGC(Object obj) { return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInlineInGC(Object obj) { UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); return Word.objectToUntrackedPointer(obj).add(size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index edb8c09bb17b..838d452afd34 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -435,6 +435,13 @@ public static long abs(long a) { } } + public static class Byte { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int toUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + } + public static class Long { /** Uninterruptible version of {@link java.lang.Long#numberOfLeadingZeros(long)}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java index cb6a17539608..ce0e926ebee1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java @@ -177,6 +177,7 @@ public long blockNoTransition(long nanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { /* Nothing to do. */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java index 6a5872171303..f85d789026dd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java @@ -98,6 +98,7 @@ public void blockNoTransitionUnspecifiedOwner() { /** * Wakes up a single thread that is waiting on this condition. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { throw VMError.shouldNotReachHere("VMCondition cannot be used during native image generation"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f381b6ed1b4d..9e2be0270d71 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -103,6 +103,7 @@ public int protect(PointerBase start, UnsignedWord nbytes, EnumSet acces } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, false); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 3b6076c42a21..9815a26d16f5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -64,10 +64,12 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; +import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; @@ -559,6 +561,26 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.joinThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void closeOSThreadHandle(OSThreadHandle threadHandle) { @@ -1189,8 +1211,19 @@ static void blockedOn(Target_sun_nio_ch_Interruptible b) { } } + @RawStructure public interface OSThreadHandle extends PointerBase { } + + @RawPointerTo(OSThreadHandle.class) + public interface OSThreadHandlePointer extends PointerBase { + void write(int index, OSThreadHandle value); + + OSThreadHandle read(int index); + } + + public interface ThreadLocalKey extends PointerBase { + } } @TargetClass(value = ThreadPoolExecutor.class, innerClass = "Worker") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 03224d272fea..4d4b7d0ba26b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -896,7 +896,7 @@ public static int getSafepointBehaviorVolatile(IsolateThread vmThread) { * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * thread will also actively prevent the VM from reaching a safepoint (regardless of its * thread status). - * + * * NOTE: Be careful with this method and make sure that this thread does not allocate any * Java objects as this could result deadlocks. This method will only prevent safepoints * reliably if it is called from a thread with {@link StatusSupport#STATUS_IN_JAVA}. @@ -910,7 +910,7 @@ public static void preventSafepoints() { /** * Marks the thread as crashed. This method may only be used in places where it is not * possible to safely detach a thread. - * + * * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * safepoint handling will ignore the thread so that the VM can reach a safepoint regardless * of the status of this thread. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java index 22f004c74e27..43665437715d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java @@ -26,6 +26,8 @@ import org.graalvm.word.Pointer; +import com.oracle.svm.core.Uninterruptible; + public class TypedMemoryReader { public static int getS1(Pointer ptr) { return ptr.readByte(0); @@ -35,6 +37,7 @@ public static int getS2(Pointer ptr) { return ptr.readShort(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static int getS4(Pointer ptr) { return ptr.readInt(0); } @@ -51,6 +54,7 @@ public static int getU2(Pointer ptr) { return getS2(ptr) & 0xFFFF; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getU4(Pointer ptr) { return getS4(ptr) & 0xFFFFFFFFL; }