diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/WriteBarrierSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/WriteBarrierSnippets.java index 6ac1ee537db4..9a3252857ba3 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/WriteBarrierSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/WriteBarrierSnippets.java @@ -62,7 +62,7 @@ protected static void verifyNotArray(Object object) { } } - protected static Word getPointerToFirstArrayElement(Word address, long length, int elementStride) { + public static Word getPointerToFirstArrayElement(Word address, long length, int elementStride) { long result = address.rawValue(); if (probability(NOT_LIKELY_PROBABILITY, elementStride < 0)) { // the address points to the place after the last array element @@ -71,7 +71,7 @@ protected static Word getPointerToFirstArrayElement(Word address, long length, i return Word.unsigned(result); } - protected static Word getPointerToLastArrayElement(Word address, long length, int elementStride) { + public static Word getPointerToLastArrayElement(Word address, long length, int elementStride) { long result = address.rawValue(); if (probability(NOT_LIKELY_PROBABILITY, elementStride < 0)) { // the address points to the place after the last array element diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java index 4b3424d0dbfd..64d9f7be99d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java @@ -27,8 +27,6 @@ import java.util.ArrayList; import java.util.List; -import org.graalvm.word.UnsignedWord; - import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.image.ImageHeap; import com.oracle.svm.core.image.ImageHeapObject; @@ -78,14 +76,21 @@ public boolean isWritable() { } static final class UnalignedChunk extends Chunk { - UnalignedChunk(long begin, long endOffset, boolean writable) { + private final long objectSize; + + UnalignedChunk(long begin, long endOffset, boolean writable, long objectSize) { super(begin, endOffset, writable); + this.objectSize = objectSize; } @Override public long getTopOffset() { return getEndOffset(); } + + public long getObjectSize() { + return objectSize; + } } final class AlignedChunk extends Chunk { @@ -162,7 +167,6 @@ public long getUnallocatedBytes() { private final int alignedChunkSize; private final int alignedChunkAlignment; private final int alignedChunkObjectsOffset; - private final int unalignedChunkObjectsOffset; private long position; @@ -177,7 +181,6 @@ public long getUnallocatedBytes() { this.alignedChunkSize = UnsignedUtils.safeToInt(HeapParameters.getAlignedHeapChunkSize()); this.alignedChunkAlignment = UnsignedUtils.safeToInt(HeapParameters.getAlignedHeapChunkAlignment()); this.alignedChunkObjectsOffset = UnsignedUtils.safeToInt(AlignedHeapChunk.getObjectsStartOffset()); - this.unalignedChunkObjectsOffset = UnsignedUtils.safeToInt(UnalignedHeapChunk.getObjectStartOffset()); this.position = position; @@ -196,11 +199,11 @@ public void alignBetweenChunks(int multiple) { public long allocateUnalignedChunkForObject(ImageHeapObject obj, boolean writable) { assert currentAlignedChunk == null; - UnsignedWord objSize = Word.unsigned(obj.getSize()); - long chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objSize).rawValue(); + long objSize = obj.getSize(); + long chunkSize = UnalignedHeapChunk.getChunkSizeForObject(Word.unsigned(objSize)).rawValue(); long chunkBegin = allocateRaw(chunkSize); - unalignedChunks.add(new UnalignedChunk(chunkBegin, chunkSize, writable)); - return chunkBegin + unalignedChunkObjectsOffset; + unalignedChunks.add(new UnalignedChunk(chunkBegin, chunkSize, writable, objSize)); + return chunkBegin + UnsignedUtils.safeToInt(UnalignedHeapChunk.calculateObjectStartOffset(Word.unsigned(objSize))); } public void maybeStartAlignedChunk() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java index f2afda6ff5a8..4a25f7d4c9ff 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java @@ -243,9 +243,9 @@ private static void writeHeader(ImageHeapChunkWriter writer, Chunk previous, Chu writer.initializeAlignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext); writer.enableRememberedSetForAlignedChunk(chunkPosition, aligned.getObjects()); } else { - assert current instanceof UnalignedChunk; - writer.initializeUnalignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext); - writer.enableRememberedSetForUnalignedChunk(chunkPosition); + UnalignedChunk unalignedChunk = (UnalignedChunk) current; + writer.initializeUnalignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext, unalignedChunk.getObjectSize()); + writer.enableRememberedSetForUnalignedChunk(chunkPosition, unalignedChunk.getObjectSize()); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java index 20dc53c42432..3803769e8341 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java @@ -148,8 +148,8 @@ void appendChunk(AlignedHeapChunk.AlignedHeader hdr) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) { - RememberedSet.get().walkDirtyObjects(space.getFirstAlignedHeapChunk(), space.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, true); + void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor) { + RememberedSet.get().walkDirtyObjects(space.getFirstAlignedHeapChunk(), space.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, refVisitor, true); } @Override @@ -350,7 +350,7 @@ private void fixupReferencesBeforeCompaction(ChunkReleaser chunkReleaser, Timers private void fixupImageHeapRoots(ImageHeapInfo info) { if (HeapImpl.usesImageHeapCardMarking()) { // Note that cards have already been cleaned and roots re-marked during the initial scan - GCImpl.walkDirtyImageHeapChunkRoots(info, fixupVisitor, false); + GCImpl.walkDirtyImageHeapChunkRoots(info, fixupVisitor, refFixupVisitor, false); } else { GCImpl.walkImageHeapRoots(info, fixupVisitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java index 4df260b6cf91..fe45dc236f35 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java @@ -159,8 +159,8 @@ void swapSpaces() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) { - RememberedSet.get().walkDirtyObjects(toSpace.getFirstAlignedHeapChunk(), toSpace.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, true); + void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor) { + RememberedSet.get().walkDirtyObjects(toSpace.getFirstAlignedHeapChunk(), toSpace.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, refVisitor, true); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 29cc70f49a70..15793380c44e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -83,6 +83,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; import com.oracle.svm.core.heap.SuspendSerialGCMaxHeapSize; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.interpreter.InterpreterSupport; @@ -909,12 +910,12 @@ private void blackenDirtyImageHeapChunkRoots(ImageHeapInfo info) { * difference after references to the runtime heap were nulled, which is assumed to be rare. */ boolean clean = completeCollection; - walkDirtyImageHeapChunkRoots(info, greyToBlackObjectVisitor, clean); + walkDirtyImageHeapChunkRoots(info, greyToBlackObjectVisitor, greyToBlackObjRefVisitor, clean); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, boolean clean) { - RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, clean); + static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { + RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, refVisitor, clean); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -963,7 +964,7 @@ private void blackenDirtyCardRoots() { * Walk old generation looking for dirty cards, and within those for old-to-young * pointers. Promote any referenced young objects. */ - HeapImpl.getHeapImpl().getOldGeneration().blackenDirtyCardRoots(greyToBlackObjectVisitor); + HeapImpl.getHeapImpl().getOldGeneration().blackenDirtyCardRoots(greyToBlackObjectVisitor, greyToBlackObjRefVisitor); } finally { blackenDirtyCardRootsTimer.stop(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 63f8656ff4ae..3a84daa13fdb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -31,8 +33,8 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; -import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.log.Log; import jdk.graal.compiler.word.Word; @@ -46,7 +48,7 @@ * Since this visitor is used during collection, one instance of it is constructed during native * image generation. */ -final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { +public final class GreyToBlackObjRefVisitor implements UninterruptibleObjectReferenceVisitor { private final Counters counters; @Platforms(Platform.HOSTED_ONLY.class) @@ -60,7 +62,7 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { Pointer pos = firstObjRef; Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); @@ -71,7 +73,7 @@ public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int r } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { assert !objRef.isNull(); counters.noteObjRef(); @@ -98,13 +100,13 @@ private void visitObjectReference(Pointer objRef, boolean compressed, Object hol // Update the reference to point to the forwarded Object. Object obj = ohi.getForwardedObject(p, header); ReferenceAccess.singleton().writeObjectAt(objRef, obj, compressed); - RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); + RememberedSet.get().dirtyCardIfNecessary(holderObject, obj, objRef); return; } Object obj = p.toObjectNonNull(); if (SerialGCOptions.useCompactingOldGen() && ObjectHeaderImpl.isMarkedHeader(header)) { - RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); + RememberedSet.get().dirtyCardIfNecessary(holderObject, obj, objRef); return; } @@ -120,7 +122,7 @@ private void visitObjectReference(Pointer objRef, boolean compressed, Object hol // The reference will not be updated if a whole chunk is promoted. However, we still // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy, objRef); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index fa6abfd0dd6b..8d28931ae8f9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -233,7 +233,7 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); UnalignedHeader result = (UnalignedHeader) ChunkBasedCommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); - UnalignedHeapChunk.initialize(result, chunkSize); + UnalignedHeapChunk.initialize(result, chunkSize, objectSize); assert objectSize.belowOrEqual(HeapChunk.availableObjectMemory(result)) : "UnalignedHeapChunk insufficient for requested object"; /* Avoid zapping if unaligned chunks are pre-zeroed. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HostedImageHeapChunkWriter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HostedImageHeapChunkWriter.java index a8fddc36aba3..06c862f48fcf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HostedImageHeapChunkWriter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HostedImageHeapChunkWriter.java @@ -27,7 +27,6 @@ import java.nio.ByteBuffer; import java.util.List; -import jdk.graal.compiler.core.common.NumUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -39,6 +38,9 @@ import com.oracle.svm.core.util.HostedByteBufferPointer; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.word.Word; + @Platforms(Platform.HOSTED_ONLY.class) final class HostedImageHeapChunkWriter implements ImageHeapChunkWriter { private final ByteBuffer buffer; @@ -76,9 +78,10 @@ public void initializeAlignedChunk(int chunkPosition, long topOffset, long endOf } @Override - public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk) { + public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk, long objectSize) { int chunkOffset = getChunkOffsetInBuffer(chunkPosition); writeHeader(chunkOffset, topOffset, endOffset, offsetToPreviousChunk, offsetToNextChunk); + UnalignedHeapChunk.initialize(new HostedByteBufferPointer(buffer, chunkOffset), Word.unsigned(objectSize)); } private void writeHeader(int chunkOffset, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk) { @@ -99,9 +102,9 @@ public void enableRememberedSetForAlignedChunk(int chunkPosition, List objects); - void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk); + void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk, long objectSize); - void enableRememberedSetForUnalignedChunk(int chunkPosition); + void enableRememberedSetForUnalignedChunk(int chunkPosition, long objectSize); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index f9f6b360bb60..8c8210699f1a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -87,7 +87,7 @@ static void walkPartitionInline(Object firstObject, Object lastObject, ObjectVis Pointer base = Heap.getHeap().getImageHeapStart(); Pointer offset = current.subtract(base); UnsignedWord chunkOffset = alignedChunks ? UnsignedUtils.roundDown(offset, HeapParameters.getAlignedHeapChunkAlignment()) - : offset.subtract(UnalignedHeapChunk.getObjectStartOffset()); + : offset.subtract(UnalignedHeapChunk.getOffsetForObject(current)); HeapChunk.Header currentChunk = (HeapChunk.Header) chunkOffset.add(base); // Assumption: the order of chunks in their linked list is the same order as in memory, diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index fb88d87e5430..ee76c1f5e691 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -46,7 +46,7 @@ public abstract class OldGeneration extends Generation { abstract void beginPromotion(boolean incrementalGc); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor); + abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) abstract boolean scanGreyObjects(boolean incrementalGc); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 4a150065783b..57be28d88ff3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.heap.ReferenceInternals.getReferentFieldAddress; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; @@ -124,7 +125,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Object refObject = referentAddr.toObjectNonNull(); if (willSurviveThisCollection(refObject)) { // Either an object that got promoted without being moved or an object in the old gen. - RememberedSet.get().dirtyCardIfNecessary(dr, refObject); + RememberedSet.get().dirtyCardIfNecessary(dr, refObject, getReferentFieldAddress(dr)); return; } if (!softReferencesAreWeak && dr instanceof SoftReference) { @@ -216,7 +217,7 @@ private static boolean processRememberedRef(Reference dr) { } Object refObject = refPointer.toObjectNonNull(); if (willSurviveThisCollection(refObject)) { - RememberedSet.get().dirtyCardIfNecessary(dr, refObject); + RememberedSet.get().dirtyCardIfNecessary(dr, refObject, getReferentFieldAddress(dr)); return true; } /* diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java index 5880aa00d186..505dfee7cf32 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java @@ -27,13 +27,13 @@ import java.nio.ByteBuffer; import java.util.List; -import jdk.graal.compiler.word.Word; import org.graalvm.word.Pointer; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.image.ImageHeapObject; +import jdk.graal.compiler.word.Word; import sun.nio.ch.DirectBuffer; /** Chunk writer that uses the same methods as memory management during image runtime. */ @@ -63,7 +63,7 @@ public void initializeAlignedChunk(int chunkPosition, long topOffset, long endOf } @Override - public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk) { + public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk, long objectSize) { UnalignedHeapChunk.UnalignedHeader header = (UnalignedHeapChunk.UnalignedHeader) getChunkPointerInBuffer(chunkPosition); header.setTopOffset(Word.unsigned(topOffset)); header.setEndOffset(Word.unsigned(endOffset)); @@ -71,6 +71,8 @@ public void initializeUnalignedChunk(int chunkPosition, long topOffset, long end header.setOffsetToPreviousChunk(Word.unsigned(offsetToPreviousChunk)); header.setOffsetToNextChunk(Word.unsigned(offsetToNextChunk)); header.setIdentityHashSalt(Word.zero(), IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); + + UnalignedHeapChunk.initializeObjectStartOffset(header, Word.unsigned(objectSize)); } @Override @@ -80,7 +82,7 @@ public void enableRememberedSetForAlignedChunk(int chunkPosition, List - * +=================+-------+-------------------------------------+ - * | UnalignedHeader | Card | Object | - * | Fields | Table | | - * +=================+-------+-------------------------------------+ + * +=================+-------+--------------+-------------------------------------+ + * | UnalignedHeader | Card | Object Start | Object | + * | Fields | Table | Offset | | + * +=================+-------+--------------+-------------------------------------+ * * - * The HeapChunk fields can be accessed as declared fields. The size of the card table depends on - * the used {@link RememberedSet} implementation and may even be zero. - * - * In this implementation, I am only implementing imprecise card remembered sets, so I only need one - * entry for the whole Object. But for consistency I am treating it as a 1-element table. + * The HeapChunk fields can be accessed as declared fields. The card table and the field for the + * object start offset are optional. Whether these fields are present depends on the used + * {@link RememberedSet} implementation. The size of the card table also depends on the used + * {@link RememberedSet} implementation and may even be zero. If present, the object start offset is + * properly aligned and stored as {@link UnsignedWord}. It is needed to get the chunk address from + * the object's address. */ public final class UnalignedHeapChunk { private UnalignedHeapChunk() { // all static @@ -82,21 +87,29 @@ private UnalignedHeapChunk() { // all static /** * Additional fields beyond what is in {@link HeapChunk.Header}. * - * This does not include the card remembered set table and certainly does not include - * the object. Those fields are accessed via Pointers that are computed below. + * This does not include the card table or the object start offset. See the comment on + * the class level above. Those fields are accessed via Pointers that are computed below. */ @RawStructure public interface UnalignedHeader extends HeapChunk.Header { } - public static void initialize(UnalignedHeader chunk, UnsignedWord chunkSize) { + @Platforms(Platform.HOSTED_ONLY.class) + public static void initialize(HostedByteBufferPointer chunk, UnsignedWord objectSize) { + UnsignedWord objectStartOffset = calculateObjectStartOffset(objectSize); + RememberedSet.get().setObjectStartOffsetOfUnalignedChunk(chunk, objectStartOffset); + } + + public static void initialize(UnalignedHeader chunk, UnsignedWord chunkSize, UnsignedWord objectSize) { assert chunk.isNonNull(); - HeapChunk.initialize(chunk, UnalignedHeapChunk.getObjectStart(chunk), chunkSize); + UnsignedWord objectStartOffset = calculateObjectStartOffset(objectSize); + HeapChunk.initialize(chunk, HeapChunk.asPointer(chunk).add(objectStartOffset), chunkSize); + setObjectStartOffset(chunk, objectStartOffset); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static Pointer getObjectStart(UnalignedHeader that) { - return HeapChunk.asPointer(that).add(getObjectStartOffset()); + return HeapChunk.asPointer(that).add(getObjectStartOffset(that)); } public static Pointer getObjectEnd(UnalignedHeader that) { @@ -104,12 +117,12 @@ public static Pointer getObjectEnd(UnalignedHeader that) { } static UnsignedWord getChunkSizeForObject(UnsignedWord objectSize) { - UnsignedWord objectStart = getObjectStartOffset(); + UnsignedWord objectStart = RememberedSet.get().getHeaderSizeOfUnalignedChunk(objectSize); UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); return UnsignedUtils.roundUp(objectStart.add(objectSize), alignment); } - /** Allocate uninitialized memory within this AlignedHeapChunk. */ + /** Allocate uninitialized memory within this UnalignedHeapChunk. */ @Uninterruptible(reason = "Returns uninitialized memory.", callerMustBe = true) public static Pointer allocateMemory(UnalignedHeader that, UnsignedWord size) { UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -122,38 +135,56 @@ public static Pointer allocateMemory(UnalignedHeader that, UnsignedWord size) { return result; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static UnalignedHeader getEnclosingChunk(Object obj) { Pointer objPointer = Word.objectToUntrackedPointer(obj); return getEnclosingChunkFromObjectPointer(objPointer); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) static UnalignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { if (!GraalDirectives.inIntrinsic()) { assert HeapImpl.isImageHeapAligned() || !HeapImpl.getHeapImpl().isInImageHeap(ptr) : "can't be used for the image heap because the image heap is not aligned to the chunk size"; } - Pointer chunkPointer = ptr.subtract(getObjectStartOffset()); + Pointer chunkPointer = ptr.subtract(getOffsetForObject(ptr)); return (UnalignedHeader) chunkPointer; } + public static void initializeObjectStartOffset(UnalignedHeader that, UnsignedWord objectSize) { + UnsignedWord objectStartOffset = calculateObjectStartOffset(objectSize); + setObjectStartOffset(that, objectStartOffset); + } + + public static UnsignedWord calculateObjectStartOffset(UnsignedWord objectSize) { + return RememberedSet.get().getHeaderSizeOfUnalignedChunk(objectSize); + } + + public static void setObjectStartOffset(UnalignedHeader that, UnsignedWord objectStartOffset) { + RememberedSet.get().setObjectStartOffsetOfUnalignedChunk(that, objectStartOffset); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getObjectStartOffset(UnalignedHeader that) { + return RememberedSet.get().getObjectStartOffsetOfUnalignedChunk(that); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getOffsetForObject(Pointer objPtr) { + return RememberedSet.get().getOffsetForObjectInUnalignedChunk(objPtr); + } + public static void walkObjects(UnalignedHeader that, ObjectVisitor visitor) { HeapChunk.walkObjectsFrom(that, getObjectStart(that), visitor); } @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void walkObjectsInline(UnalignedHeader that, UninterruptibleObjectVisitor visitor) { HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } - @Fold - static UnsignedWord getObjectStartOffset() { - return RememberedSet.get().getHeaderSizeOfUnalignedChunk(); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static UnsignedWord getCommittedObjectMemory(UnalignedHeader that) { - return HeapChunk.getEndOffset(that).subtract(getObjectStartOffset()); + return HeapChunk.getEndOffset(that).subtract(getObjectStartOffset(that)); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java index 9ec7fb1cfb24..54a63d54f766 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java @@ -34,8 +34,8 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.HeapImpl; import com.oracle.svm.core.genscavenge.ObjectHeaderImpl; -import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import jdk.graal.compiler.word.Word; @@ -43,7 +43,7 @@ * Updates each reference after marking and before compaction to point to the referenced object's * future location. */ -public final class ObjectRefFixupVisitor implements ObjectReferenceVisitor { +public final class ObjectRefFixupVisitor implements UninterruptibleObjectReferenceVisitor { @Override @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index d6a969dc15cb..6ba052cbadc8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -24,11 +24,19 @@ */ package com.oracle.svm.core.genscavenge.graal; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; +import static jdk.graal.compiler.nodes.memory.address.AddressNode.Address; import java.util.Map; import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; @@ -53,7 +61,6 @@ import jdk.graal.compiler.graph.Node.NodeIntrinsic; import jdk.graal.compiler.nodes.BreakpointNode; import jdk.graal.compiler.nodes.NamedLocationIdentity; -import jdk.graal.compiler.nodes.extended.BranchProbabilityNode; import jdk.graal.compiler.nodes.extended.FixedValueAnchorNode; import jdk.graal.compiler.nodes.extended.ForeignCallNode; import jdk.graal.compiler.nodes.gc.SerialArrayRangeWriteBarrierNode; @@ -68,52 +75,86 @@ import jdk.graal.compiler.replacements.SnippetTemplate.Arguments; import jdk.graal.compiler.replacements.SnippetTemplate.SnippetInfo; import jdk.graal.compiler.replacements.Snippets; +import jdk.graal.compiler.replacements.gc.WriteBarrierSnippets; +import jdk.graal.compiler.word.Word; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaType; +/** + * Instances and hybrid objects like pods (in contrast to arrays) are always in aligned chunks, + * except for {@link StoredContinuation} objects, but these are immutable and do not need barriers. + * + * Imprecise card marking means that the card corresponding to the object start is dirtied. This is + * used for all objects in aligned chunks. + * + * Precise card marking means that the card corresponding to the actual write address is dirtied. + * This is only used for object arrays in unaligned chunks. + */ public class BarrierSnippets extends SubstrateTemplates implements Snippets { /** A LocationIdentity to distinguish card locations from other locations. */ public static final LocationIdentity CARD_REMEMBERED_SET_LOCATION = NamedLocationIdentity.mutable("CardRememberedSet"); private static final SnippetRuntime.SubstrateForeignCallDescriptor POST_WRITE_BARRIER = SnippetRuntime.findForeignCall(BarrierSnippets.class, "postWriteBarrierStub", NO_SIDE_EFFECT, CARD_REMEMBERED_SET_LOCATION); + private static final SnippetRuntime.SubstrateForeignCallDescriptor ARRAY_RANGE_POST_WRITE_BARRIER = SnippetRuntime.findForeignCall(BarrierSnippets.class, "arrayRangePostWriteBarrierStub", + NO_SIDE_EFFECT, CARD_REMEMBERED_SET_LOCATION); private final SnippetInfo postWriteBarrierSnippet; + private final SnippetInfo arrayRangePostWriteBarrierSnippet; BarrierSnippets(OptionValues options, Providers providers) { super(options, providers); this.postWriteBarrierSnippet = snippet(providers, BarrierSnippets.class, "postWriteBarrierSnippet", CARD_REMEMBERED_SET_LOCATION); + this.arrayRangePostWriteBarrierSnippet = snippet(providers, BarrierSnippets.class, "arrayRangePostWriteBarrierSnippet", CARD_REMEMBERED_SET_LOCATION); } public void registerLowerings(MetaAccessProvider metaAccess, Map, NodeLoweringProvider> lowerings) { - PostWriteBarrierLowering lowering = new PostWriteBarrierLowering(metaAccess); - lowerings.put(SerialWriteBarrierNode.class, lowering); - // write barriers are currently always imprecise - lowerings.put(SerialArrayRangeWriteBarrierNode.class, lowering); + PostWriteBarrierLowering postWriteBarrierLowering = new PostWriteBarrierLowering(metaAccess); + lowerings.put(SerialWriteBarrierNode.class, postWriteBarrierLowering); + + ArrayRangePostWriteBarrierLowering arrayRangePostWriteBarrierLowering = new ArrayRangePostWriteBarrierLowering(metaAccess); + lowerings.put(SerialArrayRangeWriteBarrierNode.class, arrayRangePostWriteBarrierLowering); } public static void registerForeignCalls(SubstrateForeignCallsProvider provider) { provider.register(POST_WRITE_BARRIER); + provider.register(ARRAY_RANGE_POST_WRITE_BARRIER); } @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void postWriteBarrierStub(Object object) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void postWriteBarrierStub(Object object, Pointer address) { ObjectHeader oh = Heap.getHeap().getObjectHeader(); UnsignedWord objectHeader = oh.readHeaderFromObject(object); if (ObjectHeaderImpl.isUnalignedHeader(objectHeader)) { - RememberedSet.get().dirtyCardForUnalignedObject(object, false); + RememberedSet.get().dirtyCardForUnalignedObject(object, address, false); } else { RememberedSet.get().dirtyCardForAlignedObject(object, false); } } + @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void arrayRangePostWriteBarrierStub(Object object, Pointer startAddress, Pointer endAddress) { + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + UnsignedWord objectHeader = oh.readHeaderFromObject(object); + if (ObjectHeaderImpl.isUnalignedHeader(objectHeader)) { + RememberedSet.get().dirtyCardRangeForUnalignedObject(object, startAddress, endAddress); + } else { + // Arrays in aligned chunks are always marked imprecise. + RememberedSet.get().dirtyCardForAlignedObject(object, false); + } + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native void callPostWriteBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object, Pointer address); + @NodeIntrinsic(ForeignCallNode.class) - private static native void callPostWriteBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object); + private static native void callArrayRangePostWriteBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object, Pointer startAddress, Pointer endAddress); @Snippet - public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean shouldOutline, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean eliminated) { + public static void postWriteBarrierSnippet(Object object, Address address, @ConstantParameter boolean shouldOutline, @ConstantParameter boolean precise, @ConstantParameter boolean eliminated) { boolean shouldVerify = SerialGCOptions.VerifyWriteBarriers.getValue(); if (!shouldVerify && eliminated) { return; @@ -123,41 +164,77 @@ public static void postWriteBarrierSnippet(Object object, @ConstantParameter boo ObjectHeader oh = Heap.getHeap().getObjectHeader(); UnsignedWord objectHeader = oh.readHeaderFromObject(fixedObject); - if (shouldVerify && alwaysAlignedChunk) { + if (shouldVerify && !precise) { /* * To increase verification coverage, we do the verification before checking if a * barrier is needed at all. */ - if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, ObjectHeaderImpl.isUnalignedHeader(objectHeader))) { + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.isUnalignedHeader(objectHeader))) { BreakpointNode.breakpoint(); } - if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, fixedObject == null)) { + if (probability(SLOW_PATH_PROBABILITY, fixedObject == null)) { BreakpointNode.breakpoint(); } } boolean needsBarrier = RememberedSet.get().hasRememberedSet(objectHeader); - if (BranchProbabilityNode.probability(BranchProbabilityNode.FREQUENT_PROBABILITY, !needsBarrier)) { + if (probability(FREQUENT_PROBABILITY, !needsBarrier)) { return; } + Word addr = Word.fromAddress(address); + if (shouldOutline && !eliminated) { - callPostWriteBarrierStub(POST_WRITE_BARRIER, fixedObject); + callPostWriteBarrierStub(POST_WRITE_BARRIER, fixedObject, addr); return; } - if (!alwaysAlignedChunk) { + if (precise) { boolean unaligned = ObjectHeaderImpl.isUnalignedHeader(objectHeader); - if (BranchProbabilityNode.probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, unaligned)) { - RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, eliminated); + if (probability(NOT_LIKELY_PROBABILITY, unaligned)) { + RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, addr, eliminated); return; } } RememberedSet.get().dirtyCardForAlignedObject(fixedObject, eliminated); + + } + + @Snippet + public static void arrayRangePostWriteBarrierSnippet(Object object, Address address, long length, @ConstantParameter int elementStride, @ConstantParameter boolean shouldOutline) { + if (probability(NOT_FREQUENT_PROBABILITY, length == 0)) { + return; + } + + Object fixedObject = FixedValueAnchorNode.getObject(object); + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + UnsignedWord objectHeader = oh.readHeaderFromObject(fixedObject); + + boolean needsBarrier = RememberedSet.get().hasRememberedSet(objectHeader); + if (probability(FREQUENT_PROBABILITY, !needsBarrier)) { + return; + } + + Word addr = Word.fromAddress(address); + Word startAddress = WriteBarrierSnippets.getPointerToFirstArrayElement(addr, length, elementStride); + Word endAddress = WriteBarrierSnippets.getPointerToLastArrayElement(addr, length, elementStride); + + if (shouldOutline) { + callArrayRangePostWriteBarrierStub(ARRAY_RANGE_POST_WRITE_BARRIER, fixedObject, startAddress, endAddress); + return; + } + + boolean unaligned = ObjectHeaderImpl.isUnalignedHeader(objectHeader); + if (probability(NOT_LIKELY_PROBABILITY, unaligned)) { + RememberedSet.get().dirtyCardRangeForUnalignedObject(fixedObject, startAddress, endAddress); + return; + } + + RememberedSet.get().dirtyCardForAlignedObject(fixedObject, false); } - private class PostWriteBarrierLowering implements NodeLoweringProvider { + private class PostWriteBarrierLowering implements NodeLoweringProvider { private final ResolvedJavaType storedContinuationType; PostWriteBarrierLowering(MetaAccessProvider metaAccess) { @@ -165,47 +242,66 @@ private class PostWriteBarrierLowering implements NodeLoweringProvider { + private final ResolvedJavaType storedContinuationType; + + ArrayRangePostWriteBarrierLowering(MetaAccessProvider metaAccessProvider) { + storedContinuationType = metaAccessProvider.lookupJavaType(StoredContinuation.class); } - private static boolean tryEliminate(WriteBarrierNode barrier) { - if (barrier instanceof SerialWriteBarrierNode serialBarrier) { - return serialBarrier.isEliminated() || serialBarrier.getBaseStatus() == SerialWriteBarrierNode.BaseStatus.NO_LOOP_OR_SAFEPOINT; - } - return false; + @Override + public void lower(SerialArrayRangeWriteBarrierNode barrier, LoweringTool tool) { + Arguments args = new Arguments(arrayRangePostWriteBarrierSnippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress(); + + ResolvedJavaType baseType = StampTool.typeOrNull(address.getBase()); + assert baseType == null || !storedContinuationType.isAssignableFrom(baseType) : "StoredContinuation should be effectively immutable and references only be written by GC"; + + args.add("object", address.getBase()); + args.add("address", address); + args.add("length", barrier.getLengthAsLong()); + args.add("elementStride", barrier.getElementStride()); + args.add("shouldOutline", shouldOutline(barrier)); + + template(tool, barrier, args).instantiate(tool.getMetaAccess(), barrier, SnippetTemplate.DEFAULT_REPLACER, args); + } + + } + + private static boolean shouldOutline(WriteBarrierNode barrier) { + if (SerialGCOptions.OutlineWriteBarriers.getValue() != null) { + return SerialGCOptions.OutlineWriteBarriers.getValue(); + } + if (GraalOptions.ReduceCodeSize.getValue(barrier.getOptions())) { + return true; } + // Newly allocated objects are likely young, so we can outline the execution after + // checking hasRememberedSet + return barrier instanceof SerialWriteBarrierNode serialBarrier && serialBarrier.getBaseStatus().likelyYoung(); } + + private static boolean tryEliminate(WriteBarrierNode barrier) { + if (barrier instanceof SerialWriteBarrierNode serialBarrier) { + return serialBarrier.isEliminated() || serialBarrier.getBaseStatus() == SerialWriteBarrierNode.BaseStatus.NO_LOOP_OR_SAFEPOINT; + } + return false; + } + } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index bf9738fc5427..10ee03df87f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge.remset; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.util.List; import org.graalvm.nativeimage.Platform; @@ -42,6 +44,7 @@ import com.oracle.svm.core.genscavenge.ObjectHeaderImpl; import com.oracle.svm.core.genscavenge.SerialGCOptions; import com.oracle.svm.core.genscavenge.compacting.ObjectMoveInfo; +import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.image.ImageHeapObject; @@ -50,6 +53,7 @@ import com.oracle.svm.core.util.UnsignedUtils; import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; import jdk.graal.compiler.replacements.nodes.AssertionNode; import jdk.graal.compiler.word.Word; @@ -145,6 +149,11 @@ static void dirtyCardForObject(Object object, boolean verifyOnly) { } } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void dirtyAllReferencesOf(Object obj) { + ForcedSerialPostWriteBarrier.force(OffsetAddressNode.address(obj, 0), false); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void walkDirtyObjects(AlignedHeader chunk, UninterruptibleObjectVisitor visitor, boolean clean) { Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -219,6 +228,10 @@ static boolean verify(AlignedHeader chunk) { return success; } + static boolean usePreciseCardMarking() { + return false; + } + /** Return the index of an object within the tables of a chunk. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fb5d7c943949..e1075f25fa06 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge.remset; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.lang.ref.Reference; import org.graalvm.word.Pointer; @@ -81,6 +83,7 @@ final class CardTable { static final byte DIRTY_ENTRY = 0; static final byte CLEAN_ENTRY = 1; static final UnsignedWord CLEAN_WORD = Word.unsigned(0x0101010101010101L); + static final UnsignedWord DIRTY_WORD = Word.unsigned(0x0000000000000000L); private static final CardTableVerificationVisitor CARD_TABLE_VERIFICATION_VISITOR = new CardTableVerificationVisitor(); @@ -92,6 +95,11 @@ public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void dirtyTable(Pointer tableStart, UnsignedWord size) { + UnmanagedMemoryUtil.fill(tableStart, size, DIRTY_ENTRY); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); @@ -156,12 +164,16 @@ public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Point if (SerialGCOptions.VerifyRememberedSet.getValue()) { Pointer curPtr = objectsStart; while (curPtr.belowThan(objectsLimit)) { - // As we only use imprecise card marking at the moment, only the card at the address - // of the object may be dirty. Object obj = curPtr.toObjectNonNull(); + boolean precise = RememberedSet.get().usePreciseCardMarking(obj); + /* + * Objects using precise card marking need to be visited, as any of their cards may + * be dirty. For objects using imprecise card marking, only the card at the address + * of the object may be dirty. + */ UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart)); - if (isClean(cardTableStart, cardTableIndex)) { - CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart); + if (precise || isClean(cardTableStart, cardTableIndex)) { + CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart, precise); InteriorObjRefWalker.walkObject(obj, CARD_TABLE_VERIFICATION_VISITOR); success &= CARD_TABLE_VERIFICATION_VISITOR.success; CARD_TABLE_VERIFICATION_VISITOR.reset(); @@ -171,7 +183,7 @@ public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Point // The referent field of java.lang.Reference is excluded from the reference // map, so we need to verify it separately. Reference ref = (Reference) obj; - success &= verifyReferent(ref, cardTableStart, objectsStart); + success &= verifyReferent(ref, cardTableStart, objectsStart, precise); } } curPtr = LayoutEncoding.getObjectEndInGC(obj); @@ -191,11 +203,11 @@ public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Point return success; } - private static boolean verifyReferent(Reference ref, Pointer cardTableStart, Pointer objectsStart) { - return verifyReference(ref, cardTableStart, objectsStart, ReferenceInternals.getReferentFieldAddress(ref), ReferenceInternals.getReferentPointer(ref)); + private static boolean verifyReferent(Reference ref, Pointer cardTableStart, Pointer objectsStart, boolean precise) { + return verifyReference(ref, cardTableStart, objectsStart, ReferenceInternals.getReferentFieldAddress(ref), ReferenceInternals.getReferentPointer(ref), precise); } - private static boolean verifyReference(Object parentObject, Pointer cardTableStart, Pointer objectsStart, Pointer reference, Pointer referencedObject) { + private static boolean verifyReference(Object parentObject, Pointer cardTableStart, Pointer objectsStart, Pointer reference, Pointer referencedObject, boolean precise) { if (referencedObject.isNull() || HeapImpl.getHeapImpl().isInImageHeap(referencedObject) || HeapImpl.getHeapImpl().isInImageHeap(parentObject) && !HeapImpl.usesImageHeapCardMarking()) { @@ -204,6 +216,12 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta Object obj = referencedObject.toObject(); HeapChunk.Header objChunk = HeapChunk.getEnclosingHeapChunk(obj); + + // If the object is marked precise, we need to check the card of the reference. + if (precise && !isClean(cardTableStart, memoryOffsetToIndex(reference.subtract(objectsStart)))) { + return true; + } + /* Fail if we find a reference to the young generation. */ boolean fromImageHeap = HeapImpl.getHeapImpl().isInImageHeap(parentObject); if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) { @@ -225,14 +243,16 @@ private static final class CardTableVerificationVisitor implements ObjectReferen private Pointer cardTableStart; private Pointer objectsStart; private boolean success; + private boolean precise; @SuppressWarnings("hiding") - public void initialize(Object parentObject, Pointer cardTableStart, Pointer objectsStart) { + public void initialize(Object parentObject, Pointer cardTableStart, Pointer objectsStart, boolean precise) { assert this.parentObject == null && this.cardTableStart.isNull() && this.objectsStart.isNull() && !this.success; this.parentObject = parentObject; this.cardTableStart = cardTableStart; this.objectsStart = objectsStart; this.success = true; + this.precise = precise; } public void reset() { @@ -240,6 +260,7 @@ public void reset() { this.cardTableStart = Word.nullPointer(); this.objectsStart = Word.nullPointer(); this.success = false; + this.precise = false; } @Override @@ -255,7 +276,7 @@ public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int r @SuppressFBWarnings(value = {"NS_DANGEROUS_NON_SHORT_CIRCUIT"}, justification = "Non-short circuit logic is used on purpose here.") private void visitObjectReference(Pointer reference, boolean compressed) { Pointer referencedObject = ReferenceAccess.singleton().readObjectAsUntrackedPointer(reference, compressed); - success &= verifyReference(parentObject, cardTableStart, objectsStart, reference, referencedObject); + success &= verifyReference(parentObject, cardTableStart, objectsStart, reference, referencedObject, precise); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 372fa5661a75..7f85fc4f4009 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -30,6 +30,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; @@ -41,11 +42,11 @@ import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.ObjectHeaderImpl; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; -import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier; import com.oracle.svm.core.genscavenge.graal.SubstrateCardTableBarrierSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.PodReferenceMapDecoder; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.DynamicHubSupport; @@ -56,7 +57,6 @@ import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.nodes.gc.BarrierSet; -import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; import jdk.graal.compiler.word.Word; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaType; @@ -82,8 +82,31 @@ public UnsignedWord getHeaderSizeOfAlignedChunk() { } @Override - public UnsignedWord getHeaderSizeOfUnalignedChunk() { - return UnalignedChunkRememberedSet.getHeaderSize(); + public UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize) { + return UnalignedChunkRememberedSet.getHeaderSize(objectSize); + } + + @Override + @Platforms(Platform.HOSTED_ONLY.class) + public void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectStartOffset) { + UnalignedChunkRememberedSet.setObjectStartOffset(chunk, objectStartOffset); + } + + @Override + public void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset) { + UnalignedChunkRememberedSet.setObjectStartOffset(chunk, objectStartOffset); + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public UnsignedWord getObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk) { + return UnalignedChunkRememberedSet.getObjectStartOffset(chunk); + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public UnsignedWord getOffsetForObjectInUnalignedChunk(Pointer objPtr) { + return UnalignedChunkRememberedSet.getOffsetForObject(objPtr); } @Override @@ -94,66 +117,73 @@ public void enableRememberedSetForAlignedChunk(HostedByteBufferPointer chunk, in @Override @Platforms(Platform.HOSTED_ONLY.class) - public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) { - UnalignedChunkRememberedSet.enableRememberedSet(chunk); + public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectSize) { + UnalignedChunkRememberedSet.enableRememberedSet(chunk, objectSize); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj, objSize); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void dirtyCardForAlignedObject(Object object, boolean verifyOnly) { AlignedChunkRememberedSet.dirtyCardForObject(object, verifyOnly); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { - UnalignedChunkRememberedSet.dirtyCardForObject(object, verifyOnly); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void dirtyCardForUnalignedObject(Object object, Pointer address, boolean verifyOnly) { + UnalignedChunkRememberedSet.dirtyCardForObject(object, address, verifyOnly); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void dirtyCardIfNecessary(Object holderObject, Object object) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void dirtyCardRangeForUnalignedObject(Object object, Pointer startAddress, Pointer endAddress) { + UnalignedChunkRememberedSet.dirtyCardRangeForObject(object, startAddress, endAddress); + } + + @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void dirtyCardIfNecessary(Object holderObject, Object object, Pointer objRef) { if (holderObject == null || object == null) { return; } @@ -179,8 +209,8 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) { if (ObjectHeaderImpl.isAlignedObject(holderObject)) { AlignedChunkRememberedSet.dirtyCardForObject(holderObject, false); } else { - assert ObjectHeaderImpl.isUnalignedObject(holderObject) : "sanity"; - UnalignedChunkRememberedSet.dirtyCardForObject(holderObject, false); + assert ObjectHeaderImpl.isUnalignedObject(holderObject); + UnalignedChunkRememberedSet.dirtyCardForObject(holderObject, objRef, false); } } } @@ -191,7 +221,7 @@ public void dirtyAllReferencesIfNecessary(Object obj) { ObjectHeader oh = Heap.getHeap().getObjectHeader(); Word header = oh.readHeaderFromObject(obj); if (RememberedSet.get().hasRememberedSet(header) && mayContainReferences(obj)) { - ForcedSerialPostWriteBarrier.force(OffsetAddressNode.address(obj, 0), false); + dirtyAllReferencesOf(obj); } } @@ -209,9 +239,20 @@ private static boolean mayContainReferences(Object obj) { }; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void dirtyAllReferencesOf(Object obj) { + if (ObjectHeaderImpl.isAlignedObject(obj)) { + AlignedChunkRememberedSet.dirtyAllReferencesOf(obj); + } else { + assert ObjectHeaderImpl.isUnalignedObject(obj); + UnalignedChunkRememberedSet.dirtyAllReferencesOf(obj); + } + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, boolean clean) { + public void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, + UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { AlignedHeader aChunk = firstAlignedChunk; while (aChunk.isNonNull()) { AlignedChunkRememberedSet.walkDirtyObjects(aChunk, visitor, clean); @@ -220,7 +261,7 @@ public void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader fi UnalignedHeader uChunk = firstUnalignedChunk; while (uChunk.isNonNull()) { - UnalignedChunkRememberedSet.walkDirtyObjects(uChunk, visitor, clean); + UnalignedChunkRememberedSet.walkDirtyObjects(uChunk, refVisitor, clean); if (uChunk.equal(lastUnalignedChunk)) { break; } @@ -257,4 +298,14 @@ public boolean verify(UnalignedHeader firstUnalignedHeapChunk, UnalignedHeader l } return success; } + + @Override + public boolean usePreciseCardMarking(Object obj) { + if (ObjectHeaderImpl.isAlignedObject(obj)) { + return AlignedChunkRememberedSet.usePreciseCardMarking(); + } else { + assert ObjectHeaderImpl.isUnalignedObject(obj); + return UnalignedChunkRememberedSet.usePreciseCardMarking(obj); + } + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 4cb699deab07..1159f8bebe11 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -31,6 +31,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; @@ -38,6 +39,7 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.image.ImageHeapObject; import com.oracle.svm.core.util.HostedByteBufferPointer; @@ -68,12 +70,40 @@ public UnsignedWord getHeaderSizeOfAlignedChunk() { } @Override - public UnsignedWord getHeaderSizeOfUnalignedChunk() { + public UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize) { + return getHeaderSizeOfUnalignedChunk(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getHeaderSizeOfUnalignedChunk() { UnsignedWord headerSize = Word.unsigned(SizeOf.get(UnalignedHeader.class)); UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); return UnsignedUtils.roundUp(headerSize, alignment); } + @Override + @Platforms(Platform.HOSTED_ONLY.class) + public void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectStartOffset) { + assert objectStartOffset.equal(getHeaderSizeOfUnalignedChunk()); + } + + @Override + public void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset) { + assert objectStartOffset.equal(getHeaderSizeOfUnalignedChunk()); + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public UnsignedWord getObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk) { + return getHeaderSizeOfUnalignedChunk(); + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public UnsignedWord getOffsetForObjectInUnalignedChunk(Pointer objPtr) { + return getHeaderSizeOfUnalignedChunk(); + } + @Override @Platforms(Platform.HOSTED_ONLY.class) public void enableRememberedSetForAlignedChunk(HostedByteBufferPointer chunk, int chunkPosition, List objects) { @@ -82,7 +112,7 @@ public void enableRememberedSetForAlignedChunk(HostedByteBufferPointer chunk, in @Override @Platforms(Platform.HOSTED_ONLY.class) - public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) { + public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectSize) { // Nothing to do. } @@ -132,14 +162,20 @@ public void dirtyCardForAlignedObject(Object object, boolean verifyOnly) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { + public void dirtyCardForUnalignedObject(Object object, Pointer address, boolean verifyOnly) { + throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void dirtyCardRangeForUnalignedObject(Object object, Pointer startAddress, Pointer endAddress) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void dirtyCardIfNecessary(Object holderObject, Object object) { + public void dirtyCardIfNecessary(Object holderObject, Object object, Pointer objRef) { // Nothing to do. } @@ -151,7 +187,8 @@ public void dirtyAllReferencesIfNecessary(Object obj) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, boolean clean) { + public void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, + UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @@ -169,4 +206,9 @@ public boolean verify(UnalignedHeader firstUnalignedHeapChunk) { public boolean verify(UnalignedHeader firstUnalignedHeapChunk, UnalignedHeader lastUnalignedHeapChunk) { return true; } + + @Override + public boolean usePreciseCardMarking(Object obj) { + throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index 658c9ac5f397..0a0b84290752 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -31,6 +31,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; @@ -38,6 +39,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; import com.oracle.svm.core.heap.BarrierSetProvider; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.image.ImageHeapObject; import com.oracle.svm.core.util.HostedByteBufferPointer; @@ -58,8 +60,26 @@ static RememberedSet get() { /** Returns the header size of aligned chunks. */ UnsignedWord getHeaderSizeOfAlignedChunk(); - /** Returns the header size of unaligned chunks. */ - UnsignedWord getHeaderSizeOfUnalignedChunk(); + /** Returns the header size of an unaligned chunk for a given object size. */ + UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize); + + /** Sets the object start offset in the unaligned chunk. */ + @Platforms(Platform.HOSTED_ONLY.class) + void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectStartOffset); + + /** Sets the object start offset in the unaligned chunk. */ + void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset); + + /** + * Get the object start offset of an unaligned chunk. If possible, use + * {@link #getOffsetForObjectInUnalignedChunk(Pointer)} instead, as that method is faster. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + UnsignedWord getObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk); + + /** Get the object start offset of this object in its unaligned chunk. */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + UnsignedWord getOffsetForObjectInUnalignedChunk(Pointer objPtr); /** * Enables remembered set tracking for an aligned chunk and its objects. Must be called when @@ -73,7 +93,7 @@ static RememberedSet get() { * adding a new chunk to the image heap or old generation. */ @Platforms(Platform.HOSTED_ONLY.class) - void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk); + void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectSize); /** * Enables remembered set tracking for an aligned chunk and its objects. Must be called when @@ -126,7 +146,17 @@ static RememberedSet get() { */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void dirtyCardForUnalignedObject(Object object, boolean verifyOnly); + void dirtyCardForUnalignedObject(Object object, Pointer address, boolean verifyOnly); + + /** + * Marks a range within an array object as dirty. May only be called for objects for which + * remembered set tracking is enabled. This tells the GC that the object may contain references + * to another generation (from old generation to young generation, or from image heap to runtime + * heap). + */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void dirtyCardRangeForUnalignedObject(Object object, Pointer startAddress, Pointer endAddress); /** * Marks the {@code holderObject} as dirty if needed according to the location of @@ -135,7 +165,7 @@ static RememberedSet get() { */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void dirtyCardIfNecessary(Object holderObject, Object object); + void dirtyCardIfNecessary(Object holderObject, Object object, Pointer objRef); /** * If remembered set tracking is enabled for the given object, this method ensures that all @@ -149,7 +179,8 @@ static RememberedSet get() { * linked lists} of aligned and unaligned chunks. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, boolean clean); + void walkDirtyObjects(AlignedHeader firstAlignedChunk, UnalignedHeader firstUnalignedChunk, UnalignedHeader lastUnalignedChunk, UninterruptibleObjectVisitor visitor, + UninterruptibleObjectReferenceVisitor refVisitor, boolean clean); /** * Verify the remembered set for an aligned chunk and all its linked-list successors. @@ -166,4 +197,10 @@ static RememberedSet get() { * including) another unaligned chunk. */ boolean verify(UnalignedHeader firstUnalignedHeapChunk, UnalignedHeader lastUnalignedHeapChunk); + + /** + * Checks if an object uses precise card marking. This method does not check the + * remembered set bit or if the object has actually any references. + */ + boolean usePreciseCardMarking(Object obj); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index f0f2c1c33df7..967d45f67d46 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge.remset; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -36,34 +38,88 @@ import com.oracle.svm.core.genscavenge.ObjectHeaderImpl; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; -import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; +import com.oracle.svm.core.heap.ObjectHeader; +import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.hub.HubType; +import com.oracle.svm.core.hub.InteriorObjRefWalker; +import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.HostedByteBufferPointer; import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.directives.GraalDirectives; import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.nodes.java.ArrayLengthNode; import jdk.graal.compiler.replacements.nodes.AssertionNode; import jdk.graal.compiler.word.Word; final class UnalignedChunkRememberedSet { + private UnalignedChunkRememberedSet() { } - @Fold - public static UnsignedWord getHeaderSize() { - UnsignedWord headerSize = getCardTableLimitOffset(); + public static UnsignedWord getHeaderSize(UnsignedWord objectSize) { + UnsignedWord headerSize = getCardTableLimitOffset(objectSize); + headerSize = headerSize.add(sizeOfObjectStartOffsetField()); + UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); return UnsignedUtils.roundUp(headerSize, alignment); } @Platforms(Platform.HOSTED_ONLY.class) - public static void enableRememberedSet(HostedByteBufferPointer chunk) { - CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); + public static void setObjectStartOffset(HostedByteBufferPointer chunk, UnsignedWord objectStartOffset) { + chunk.writeWord(objectStartOffset.subtract(sizeOfObjectStartOffsetField()), objectStartOffset); + } + + public static void setObjectStartOffset(UnalignedHeader chunk, UnsignedWord objectStartOffset) { + HeapChunk.asPointer(chunk).writeWord(objectStartOffset.subtract(sizeOfObjectStartOffsetField()), objectStartOffset); + assert getObjectStartOffset(chunk).equal(objectStartOffset); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getObjectStartOffset(UnalignedHeader chunk) { + UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); + UnsignedWord headerSize = getCardTableStartOffset(); + UnsignedWord objectStartOffsetSize = Word.unsigned(sizeOfObjectStartOffsetField()); + UnsignedWord alignedObjectStartOffsetSize = UnsignedUtils.roundUp(objectStartOffsetSize, alignment); + UnsignedWord ctAndObjSize = chunk.getEndOffset().subtract(headerSize).subtract(alignedObjectStartOffsetSize); + + /* + * The combined card table and object size is roundUp(objSize / BYTES_COVERED_BY_ENTRY, + * alignment) + objSize. To get the object size from this combined size, the inverse needs + * to be calculated. The rounding down is needed, as the card table size is rounded up. + */ + UnsignedWord objSizeWithCtAlignment = ctAndObjSize.multiply(CardTable.BYTES_COVERED_BY_ENTRY).unsignedDivide(CardTable.BYTES_COVERED_BY_ENTRY + 1); + UnsignedWord objSize = UnsignedUtils.roundDown(objSizeWithCtAlignment, alignment); + + UnsignedWord objectStartOffset = HeapChunk.getEndOffset(chunk).subtract(objSize); + + assert objectStartOffset.equal(getOffsetForObject(HeapChunk.asPointer(chunk).add(objectStartOffset))); + + return objectStartOffset; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static UnsignedWord getOffsetForObject(Pointer objPtr) { + return objPtr.readWord(-sizeOfObjectStartOffsetField()); + } + + @Platforms(Platform.HOSTED_ONLY.class) + public static void enableRememberedSet(HostedByteBufferPointer chunk, UnsignedWord objectSize) { + CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize(objectSize)); // The remembered set bit in the header will be set by the code that writes the objects. } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { - CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); + CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize(chunk)); // Unaligned chunks don't have a first object table. Object obj = UnalignedHeapChunk.getObjectStart(chunk).toObjectNonNull(); @@ -72,18 +128,18 @@ public static void enableRememberedSet(UnalignedHeader chunk) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { - CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); + CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize(chunk)); } /** - * Dirty the card corresponding to the given Object. This has to be fast, because it is used by - * the post-write barrier. + * Dirty the card corresponding to the given address within the given object. This has to be + * fast, because it is used by the post-write barrier. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void dirtyCardForObject(Object obj, boolean verifyOnly) { + public static void dirtyCardForObject(Object obj, Pointer address, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); - UnsignedWord objectIndex = getObjectIndex(); + UnsignedWord objectIndex = CardTable.memoryOffsetToIndex(address.subtract(Word.objectToUntrackedPointer(obj))); if (verifyOnly) { AssertionNode.assertion(false, CardTable.isDirty(cardTableStart, objectIndex), "card must be dirty", "", "", 0L, 0L); } else { @@ -91,25 +147,220 @@ public static void dirtyCardForObject(Object obj, boolean verifyOnly) { } } - @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") - public static void walkDirtyObjects(UnalignedHeader chunk, UninterruptibleObjectVisitor visitor, boolean clean) { - Pointer rememberedSetStart = getCardTableStart(chunk); - UnsignedWord objectIndex = getObjectIndex(); - if (CardTable.isDirty(rememberedSetStart, objectIndex)) { + /** + * Dirty the cards corresponding to [start address, end address]. This has to be fast, because + * it is used by the array range post-write barrier. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void dirtyCardRangeForObject(Object obj, Pointer startAddress, Pointer endAddress) { + UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); + Pointer cardTableStart = getCardTableStart(chunk); + + Pointer objPtr = Word.objectToUntrackedPointer(obj); + UnsignedWord startIndex = CardTable.memoryOffsetToIndex(startAddress.subtract(objPtr)); + UnsignedWord endIndex = CardTable.memoryOffsetToIndex(endAddress.subtract(objPtr)); + + UnsignedWord curIndex = startIndex; + do { + CardTable.setDirty(cardTableStart, curIndex); + curIndex = curIndex.add(1); + } while (GraalDirectives.injectIterationCount(10, curIndex.belowOrEqual(endIndex))); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void dirtyAllReferencesOf(Object obj) { + DynamicHub hub = KnownIntrinsics.readHub(obj); + int hubType = hub.getHubType(); + + Pointer objPtr = Word.objectToUntrackedPointer(obj); + Pointer chunk = objPtr.subtract(getOffsetForObject(objPtr)); + + switch (hubType) { + case HubType.STORED_CONTINUATION_INSTANCE: + if (!ContinuationSupport.isSupported()) { + throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); + } + VMError.guarantee(StoredContinuationAccess.isInitialized((StoredContinuation) obj), "The stored continuation is still being initialized and does not contain valid stack data yet."); + + // Stored continuation objects are always marked imprecise. + CardTable.setDirty(getCardTableStart(chunk), Word.zero()); + return; + case HubType.PRIMITIVE_ARRAY: + return; + case HubType.OBJECT_ARRAY: + CardTable.dirtyTable(getCardTableStart(chunk), getCardTableSize(objPtr)); + return; + default: + throw VMError.shouldNotReachHere("Unexpected hub type."); + } + + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void walkDirtyObjects(UnalignedHeader chunk, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { + UnsignedWord objStartOffset = getObjectStartOffset(chunk); + Object obj = HeapChunk.asPointer(chunk).add(objStartOffset).toObjectNonNull(); + + DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); + + // Return as early as possible for primitive arrays. + if (objHub.getHubType() == HubType.PRIMITIVE_ARRAY) { + return; + } + + Pointer cardTableStart = getCardTableStart(chunk); + assert cardTableStart.unsignedRemainder(wordSize()).equal(0); + + switch (objHub.getHubType()) { + case HubType.STORED_CONTINUATION_INSTANCE: + walkStoredContinuationImprecise((StoredContinuation) obj, cardTableStart, refVisitor, clean); + return; + case HubType.OBJECT_ARRAY: + UnsignedWord cardTableLimitIdx = objStartOffset.subtract(sizeOfObjectStartOffsetField()).subtract(getCardTableStartOffset()); + walkObjectArrayPrecise(obj, cardTableStart, cardTableLimitIdx, refVisitor, clean); + return; + default: + throw VMError.shouldNotReachHere("Unexpected hub type."); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void walkStoredContinuationImprecise(StoredContinuation s, Pointer cardTableStart, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { + if (!ContinuationSupport.isSupported()) { + throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); + } else if (StoredContinuationAccess.shouldWalkContinuation(s)) { + /* + * Stored continuations are always marked imprecise, so only the first card needs to be + * checked. + */ + if (CardTable.isDirty(cardTableStart, Word.zero())) { + if (clean) { + CardTable.setClean(cardTableStart, Word.zero()); + } + InteriorObjRefWalker.walkObjectInline(s, refVisitor); + } + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void walkObjectArrayPrecise(Object obj, Pointer cardTableStart, UnsignedWord cardTableLimitIdx, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); + + DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); + int length = ArrayLengthNode.arrayLength(obj); + int layoutEncoding = objHub.getLayoutEncoding(); + + /* + * All offsets below until the end of this method are offsets from the object start, and not + * from the chunk. + */ + UnsignedWord elementStartOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, 0); + UnsignedWord elementEndOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, length); + + UnsignedWord iOffset = elementStartOffset; + while (iOffset.belowThan(elementEndOffset)) { + UnsignedWord dirtyCardStart = findFirstDirtyCard(cardTableStart, CardTable.memoryOffsetToIndex(iOffset), cardTableLimitIdx); + UnsignedWord dirtyCardEnd = findFirstCleanCard(cardTableStart, dirtyCardStart, cardTableLimitIdx, clean); + + if (dirtyCardStart.equal(dirtyCardEnd)) { + break; + } + + // Located a non-empty dirty card interval [dirtyCardStart, dirtyCardEnd). + UnsignedWord dirtyStartOffset = dirtyCardStart.multiply(CardTable.BYTES_COVERED_BY_ENTRY); + UnsignedWord dirtyEndOffset = dirtyCardEnd.multiply(CardTable.BYTES_COVERED_BY_ENTRY); + + // The object may start or end within a card. + UnsignedWord startOffset = UnsignedUtils.max(dirtyStartOffset, elementStartOffset); + UnsignedWord endOffset = UnsignedUtils.min(dirtyEndOffset, elementEndOffset); + + Pointer refPtr = Word.objectToUntrackedPointer(obj).add(startOffset); + UnsignedWord nReferences = (endOffset.subtract(startOffset)).unsignedDivide(referenceSize); + refVisitor.visitObjectReferences(refPtr, isCompressed, referenceSize, obj, UnsignedUtils.safeToInt(nReferences)); + + iOffset = dirtyEndOffset; + } + } + + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+16/src/hotspot/share/gc/g1/g1RemSet.cpp#L562-L586") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord findFirstDirtyCard(Pointer ctAdr, UnsignedWord startIdx, UnsignedWord endIdx) { + assert UnsignedUtils.isAMultiple(endIdx, Word.unsigned(wordSize())); + + UnsignedWord wordSize = Word.unsigned(wordSize()); + UnsignedWord curIdx = startIdx; + + while (!UnsignedUtils.isAMultiple(curIdx, wordSize)) { + if (CardTable.isDirty(ctAdr, curIdx)) { + return curIdx; + } + curIdx = curIdx.add(1); + } + + for (/* empty */; curIdx.belowThan(endIdx); curIdx = curIdx.add(wordSize)) { + UnsignedWord wordValue = ctAdr.readWord(curIdx); + + if (wordValue.notEqual(CardTable.CLEAN_WORD)) { + for (int i = 0; i < wordSize(); i++) { + if (CardTable.isDirty(ctAdr, curIdx)) { + return curIdx; + } + curIdx = curIdx.add(1); + } + VMError.shouldNotReachHere("should have returned early"); + } + } + + return endIdx; + } + + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+16/src/hotspot/share/gc/g1/g1RemSet.cpp#L588-L612") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord findFirstCleanCard(Pointer ctAdr, UnsignedWord startIdx, UnsignedWord endIdx, boolean clean) { + assert UnsignedUtils.isAMultiple(endIdx, Word.unsigned(wordSize())); + + UnsignedWord wordSize = Word.unsigned(wordSize()); + UnsignedWord curIdx = startIdx; + + while (!UnsignedUtils.isAMultiple(curIdx, wordSize)) { + if (!CardTable.isDirty(ctAdr, curIdx)) { + return curIdx; + } if (clean) { - CardTable.setClean(rememberedSetStart, objectIndex); + CardTable.setClean(ctAdr, curIdx); } + curIdx = curIdx.add(1); + } + + for (/* empty */; curIdx.belowThan(endIdx); curIdx = curIdx.add(wordSize)) { + UnsignedWord wordValue = ctAdr.readWord(curIdx); - Pointer objectsStart = UnalignedHeapChunk.getObjectStart(chunk); - Object obj = objectsStart.toObjectNonNull(); - visitor.visitObject(obj); + if (wordValue.notEqual(CardTable.DIRTY_WORD)) { + for (int i = 0; i < wordSize(); i++) { + if (!CardTable.isDirty(ctAdr, curIdx)) { + return curIdx; + } + curIdx = curIdx.add(1); + } + VMError.shouldNotReachHere("should have early-returned"); + } + if (clean) { + ctAdr.writeWord(curIdx, CardTable.CLEAN_WORD); + } } + + return endIdx; } public static boolean verify(UnalignedHeader chunk) { return CardTable.verify(getCardTableStart(chunk), getCardTableEnd(chunk), UnalignedHeapChunk.getObjectStart(chunk), HeapChunk.getTopPointer(chunk)); } + public static boolean usePreciseCardMarking(Object obj) { + return !(obj instanceof StoredContinuation); + } + @Fold static UnsignedWord getCardTableStartOffset() { UnsignedWord headerSize = Word.unsigned(SizeOf.get(UnalignedHeader.class)); @@ -117,39 +368,53 @@ static UnsignedWord getCardTableStartOffset() { return UnsignedUtils.roundUp(headerSize, alignment); } - @Fold - static UnsignedWord getCardTableSize() { - UnsignedWord requiredSize = CardTable.tableSizeForMemorySize(Word.unsigned(1)); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getCardTableSize(UnsignedWord objectSize) { + UnsignedWord requiredSize = CardTable.tableSizeForMemorySize(objectSize); UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); return UnsignedUtils.roundUp(requiredSize, alignment); } - @Fold - static UnsignedWord getCardTableLimitOffset() { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getCardTableSize(UnalignedHeader chunk) { + return getObjectStartOffset(chunk).subtract(sizeOfObjectStartOffsetField()).subtract(getCardTableStartOffset()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getCardTableSize(Pointer obj) { + return getOffsetForObject(obj).subtract(sizeOfObjectStartOffsetField()).subtract(getCardTableStartOffset()); + } + + private static UnsignedWord getCardTableLimitOffset(UnsignedWord objectSize) { UnsignedWord tableStart = getCardTableStartOffset(); - UnsignedWord tableSize = getCardTableSize(); + UnsignedWord tableSize = getCardTableSize(objectSize); UnsignedWord tableLimit = tableStart.add(tableSize); UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()); return UnsignedUtils.roundUp(tableLimit, alignment); } - @Fold - static UnsignedWord getObjectIndex() { - return Word.zero(); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static Pointer getCardTableEnd(UnalignedHeader chunk) { - return getCardTableStart(chunk).add(getCardTableSize()); + return getCardTableStart(chunk).add(getCardTableSize(getObjectStartOffset(chunk).subtract(sizeOfObjectStartOffsetField()))); + } + + @Fold + static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + + @Fold + static int sizeOfObjectStartOffsetField() { + return wordSize(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index 9a6e59eb4a54..7c418dbb6d2f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -116,6 +116,11 @@ public static CodePointer getIP(StoredContinuation s) { return s.ip; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static boolean isInitialized(StoredContinuation s) { + return s.ip.isNonNull(); + } + public static int allocateToYield(Target_jdk_internal_vm_Continuation c, Pointer baseSp, Pointer sp, CodePointer ip) { assert baseSp.isNonNull() && sp.isNonNull() && ip.isNonNull(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectReferenceVisitor.java new file mode 100644 index 000000000000..ffe62cc38bb6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectReferenceVisitor.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.heap; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.word.Pointer; + +import com.oracle.svm.core.Uninterruptible; + +public interface UninterruptibleObjectReferenceVisitor extends ObjectReferenceVisitor { + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/HostedByteBufferPointer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/HostedByteBufferPointer.java index 0d0019ea5ad3..c11dcc8134d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/HostedByteBufferPointer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/HostedByteBufferPointer.java @@ -26,7 +26,6 @@ import java.nio.ByteBuffer; -import jdk.graal.compiler.core.common.NumUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.ComparableWord; @@ -35,6 +34,10 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; +import com.oracle.svm.core.config.ConfigurationValues; + +import jdk.graal.compiler.core.common.NumUtil; + @Platforms(Platform.HOSTED_ONLY.class) public final class HostedByteBufferPointer implements Pointer { private final ByteBuffer buffer; @@ -501,7 +504,14 @@ public void writeDouble(int offset, double val) { @Override public void writeWord(int offset, WordBase val) { - throw unsupported(); + long value = val.rawValue(); + int wordSize = ConfigurationValues.getTarget().wordSize; + if (wordSize == Integer.BYTES) { + buffer.putInt(baseOffset + offset, NumUtil.safeToUInt(value)); + } else { + assert wordSize == Long.BYTES; + buffer.putLong(baseOffset + offset, value); + } } @Override