[GR-34010] Use precise write barriers for Object arrays.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/WriteBarrierSnippets.java

@@ -62,7 +62,7 @@ protected static void verifyNotArray(Object object) {
         }
     }
 
-    protected static Word getPointerToFirstArrayElement(Word address, long length, int elementStride) {
+    public static Word getPointerToFirstArrayElement(Word address, long length, int elementStride) {
         long result = address.rawValue();
         if (probability(NOT_LIKELY_PROBABILITY, elementStride < 0)) {
             // the address points to the place after the last array element
@@ -71,7 +71,7 @@ protected static Word getPointerToFirstArrayElement(Word address, long length, i
         return Word.unsigned(result);
     }
 
-    protected static Word getPointerToLastArrayElement(Word address, long length, int elementStride) {
+    public static Word getPointerToLastArrayElement(Word address, long length, int elementStride) {
         long result = address.rawValue();
         if (probability(NOT_LIKELY_PROBABILITY, elementStride < 0)) {
             // the address points to the place after the last array element

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java

@@ -27,8 +27,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.graalvm.word.UnsignedWord;
-
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.image.ImageHeap;
 import com.oracle.svm.core.image.ImageHeapObject;
@@ -78,14 +76,21 @@ public boolean isWritable() {
     }
 
     static final class UnalignedChunk extends Chunk {
-        UnalignedChunk(long begin, long endOffset, boolean writable) {
+        private final long objectSize;
+
+        UnalignedChunk(long begin, long endOffset, boolean writable, long objectSize) {
             super(begin, endOffset, writable);
+            this.objectSize = objectSize;
         }
 
         @Override
         public long getTopOffset() {
             return getEndOffset();
         }
+
+        public long getObjectSize() {
+            return objectSize;
+        }
     }
 
     final class AlignedChunk extends Chunk {
@@ -162,7 +167,6 @@ public long getUnallocatedBytes() {
     private final int alignedChunkSize;
     private final int alignedChunkAlignment;
     private final int alignedChunkObjectsOffset;
-    private final int unalignedChunkObjectsOffset;
 
     private long position;
 
@@ -177,7 +181,6 @@ public long getUnallocatedBytes() {
         this.alignedChunkSize = UnsignedUtils.safeToInt(HeapParameters.getAlignedHeapChunkSize());
         this.alignedChunkAlignment = UnsignedUtils.safeToInt(HeapParameters.getAlignedHeapChunkAlignment());
         this.alignedChunkObjectsOffset = UnsignedUtils.safeToInt(AlignedHeapChunk.getObjectsStartOffset());
-        this.unalignedChunkObjectsOffset = UnsignedUtils.safeToInt(UnalignedHeapChunk.getObjectStartOffset());
 
         this.position = position;
 
@@ -196,11 +199,11 @@ public void alignBetweenChunks(int multiple) {
 
     public long allocateUnalignedChunkForObject(ImageHeapObject obj, boolean writable) {
         assert currentAlignedChunk == null;
-        UnsignedWord objSize = Word.unsigned(obj.getSize());
-        long chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objSize).rawValue();
+        long objSize = obj.getSize();
+        long chunkSize = UnalignedHeapChunk.getChunkSizeForObject(Word.unsigned(objSize)).rawValue();
         long chunkBegin = allocateRaw(chunkSize);
-        unalignedChunks.add(new UnalignedChunk(chunkBegin, chunkSize, writable));
-        return chunkBegin + unalignedChunkObjectsOffset;
+        unalignedChunks.add(new UnalignedChunk(chunkBegin, chunkSize, writable, objSize));
+        return chunkBegin + UnsignedUtils.safeToInt(UnalignedHeapChunk.calculateObjectStartOffset(Word.unsigned(objSize)));
     }
 
     public void maybeStartAlignedChunk() {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java

@@ -243,9 +243,9 @@ private static void writeHeader(ImageHeapChunkWriter writer, Chunk previous, Chu
                 writer.initializeAlignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext);
                 writer.enableRememberedSetForAlignedChunk(chunkPosition, aligned.getObjects());
             } else {
-                assert current instanceof UnalignedChunk;
-                writer.initializeUnalignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext);
-                writer.enableRememberedSetForUnalignedChunk(chunkPosition);
+                UnalignedChunk unalignedChunk = (UnalignedChunk) current;
+                writer.initializeUnalignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext, unalignedChunk.getObjectSize());
+                writer.enableRememberedSetForUnalignedChunk(chunkPosition, unalignedChunk.getObjectSize());
             }
         }
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -148,8 +148,8 @@ void appendChunk(AlignedHeapChunk.AlignedHeader hdr) {
 
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) {
-        RememberedSet.get().walkDirtyObjects(space.getFirstAlignedHeapChunk(), space.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, true);
+    void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor) {
+        RememberedSet.get().walkDirtyObjects(space.getFirstAlignedHeapChunk(), space.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, refVisitor, true);
     }
 
     @Override
@@ -350,7 +350,7 @@ private void fixupReferencesBeforeCompaction(ChunkReleaser chunkReleaser, Timers
     private void fixupImageHeapRoots(ImageHeapInfo info) {
         if (HeapImpl.usesImageHeapCardMarking()) {
             // Note that cards have already been cleaned and roots re-marked during the initial scan
-            GCImpl.walkDirtyImageHeapChunkRoots(info, fixupVisitor, false);
+            GCImpl.walkDirtyImageHeapChunkRoots(info, fixupVisitor, refFixupVisitor, false);
         } else {
             GCImpl.walkImageHeapRoots(info, fixupVisitor);
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java

@@ -159,8 +159,8 @@ void swapSpaces() {
 
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) {
-        RememberedSet.get().walkDirtyObjects(toSpace.getFirstAlignedHeapChunk(), toSpace.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, true);
+    void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor) {
+        RememberedSet.get().walkDirtyObjects(toSpace.getFirstAlignedHeapChunk(), toSpace.getFirstUnalignedHeapChunk(), Word.nullPointer(), visitor, refVisitor, true);
     }
 
     @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -83,6 +83,7 @@
 import com.oracle.svm.core.heap.RestrictHeapAccess;
 import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner;
 import com.oracle.svm.core.heap.SuspendSerialGCMaxHeapSize;
+import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor;
 import com.oracle.svm.core.heap.UninterruptibleObjectVisitor;
 import com.oracle.svm.core.heap.VMOperationInfos;
 import com.oracle.svm.core.interpreter.InterpreterSupport;
@@ -909,12 +910,12 @@ private void blackenDirtyImageHeapChunkRoots(ImageHeapInfo info) {
          * difference after references to the runtime heap were nulled, which is assumed to be rare.
          */
         boolean clean = completeCollection;
-        walkDirtyImageHeapChunkRoots(info, greyToBlackObjectVisitor, clean);
+        walkDirtyImageHeapChunkRoots(info, greyToBlackObjectVisitor, greyToBlackObjRefVisitor, clean);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, boolean clean) {
-        RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, clean);
+    static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) {
+        RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, refVisitor, clean);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
@@ -963,7 +964,7 @@ private void blackenDirtyCardRoots() {
              * Walk old generation looking for dirty cards, and within those for old-to-young
              * pointers. Promote any referenced young objects.
              */
-            HeapImpl.getHeapImpl().getOldGeneration().blackenDirtyCardRoots(greyToBlackObjectVisitor);
+            HeapImpl.getHeapImpl().getOldGeneration().blackenDirtyCardRoots(greyToBlackObjectVisitor, greyToBlackObjRefVisitor);
         } finally {
             blackenDirtyCardRootsTimer.stop();
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java

@@ -24,15 +24,17 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.Pointer;
 
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
-import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
+import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor;
 import com.oracle.svm.core.log.Log;
 
 import jdk.graal.compiler.word.Word;
@@ -46,7 +48,7 @@
  * Since this visitor is used during collection, one instance of it is constructed during native
  * image generation.
  */
-final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor {
+public final class GreyToBlackObjRefVisitor implements UninterruptibleObjectReferenceVisitor {
     private final Counters counters;
 
     @Platforms(Platform.HOSTED_ONLY.class)
@@ -60,7 +62,7 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor {
 
     @Override
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) {
         Pointer pos = firstObjRef;
         Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize));
@@ -71,7 +73,7 @@ public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int r
     }
 
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) {
         assert !objRef.isNull();
         counters.noteObjRef();
@@ -98,13 +100,13 @@ private void visitObjectReference(Pointer objRef, boolean compressed, Object hol
                 // Update the reference to point to the forwarded Object.
                 Object obj = ohi.getForwardedObject(p, header);
                 ReferenceAccess.singleton().writeObjectAt(objRef, obj, compressed);
-                RememberedSet.get().dirtyCardIfNecessary(holderObject, obj);
+                RememberedSet.get().dirtyCardIfNecessary(holderObject, obj, objRef);
                 return;
             }
 
             Object obj = p.toObjectNonNull();
             if (SerialGCOptions.useCompactingOldGen() && ObjectHeaderImpl.isMarkedHeader(header)) {
-                RememberedSet.get().dirtyCardIfNecessary(holderObject, obj);
+                RememberedSet.get().dirtyCardIfNecessary(holderObject, obj, objRef);
                 return;
             }
 
@@ -120,7 +122,7 @@ private void visitObjectReference(Pointer objRef, boolean compressed, Object hol
 
             // The reference will not be updated if a whole chunk is promoted. However, we still
             // might have to dirty the card.
-            RememberedSet.get().dirtyCardIfNecessary(holderObject, copy);
+            RememberedSet.get().dirtyCardIfNecessary(holderObject, copy, objRef);
         }
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java

@@ -233,7 +233,7 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) {
         UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize);
 
         UnalignedHeader result = (UnalignedHeader) ChunkBasedCommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize);
-        UnalignedHeapChunk.initialize(result, chunkSize);
+        UnalignedHeapChunk.initialize(result, chunkSize, objectSize);
         assert objectSize.belowOrEqual(HeapChunk.availableObjectMemory(result)) : "UnalignedHeapChunk insufficient for requested object";
 
         /* Avoid zapping if unaligned chunks are pre-zeroed. */

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HostedImageHeapChunkWriter.java

@@ -27,7 +27,6 @@
 import java.nio.ByteBuffer;
 import java.util.List;
 
-import jdk.graal.compiler.core.common.NumUtil;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.struct.SizeOf;
@@ -39,6 +38,9 @@
 import com.oracle.svm.core.util.HostedByteBufferPointer;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.core.common.NumUtil;
+import jdk.graal.compiler.word.Word;
+
 @Platforms(Platform.HOSTED_ONLY.class)
 final class HostedImageHeapChunkWriter implements ImageHeapChunkWriter {
     private final ByteBuffer buffer;
@@ -76,9 +78,10 @@ public void initializeAlignedChunk(int chunkPosition, long topOffset, long endOf
     }
 
     @Override
-    public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk) {
+    public void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk, long objectSize) {
         int chunkOffset = getChunkOffsetInBuffer(chunkPosition);
         writeHeader(chunkOffset, topOffset, endOffset, offsetToPreviousChunk, offsetToNextChunk);
+        UnalignedHeapChunk.initialize(new HostedByteBufferPointer(buffer, chunkOffset), Word.unsigned(objectSize));
     }
 
     private void writeHeader(int chunkOffset, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk) {
@@ -99,9 +102,9 @@ public void enableRememberedSetForAlignedChunk(int chunkPosition, List<ImageHeap
     }
 
     @Override
-    public void enableRememberedSetForUnalignedChunk(int chunkPosition) {
+    public void enableRememberedSetForUnalignedChunk(int chunkPosition, long objectSize) {
         int chunkOffset = getChunkOffsetInBuffer(chunkPosition);
-        rememberedSet.enableRememberedSetForUnalignedChunk(new HostedByteBufferPointer(buffer, chunkOffset));
+        rememberedSet.enableRememberedSetForUnalignedChunk(new HostedByteBufferPointer(buffer, chunkOffset), Word.unsigned(objectSize));
     }
 
     static void putObjectReference(ByteBuffer buffer, int offset, long value) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapChunkWriter.java

@@ -33,7 +33,7 @@ interface ImageHeapChunkWriter {
 
     void enableRememberedSetForAlignedChunk(int chunkPosition, List<ImageHeapObject> objects);
 
-    void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk);
+    void initializeUnalignedChunk(int chunkPosition, long topOffset, long endOffset, long offsetToPreviousChunk, long offsetToNextChunk, long objectSize);
 
-    void enableRememberedSetForUnalignedChunk(int chunkPosition);
+    void enableRememberedSetForUnalignedChunk(int chunkPosition, long objectSize);
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java

@@ -87,7 +87,7 @@ static void walkPartitionInline(Object firstObject, Object lastObject, ObjectVis
         Pointer base = Heap.getHeap().getImageHeapStart();
         Pointer offset = current.subtract(base);
         UnsignedWord chunkOffset = alignedChunks ? UnsignedUtils.roundDown(offset, HeapParameters.getAlignedHeapChunkAlignment())
-                        : offset.subtract(UnalignedHeapChunk.getObjectStartOffset());
+                        : offset.subtract(UnalignedHeapChunk.getOffsetForObject(current));
         HeapChunk.Header<?> currentChunk = (HeapChunk.Header<?>) chunkOffset.add(base);
 
         // Assumption: the order of chunks in their linked list is the same order as in memory,

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java

@@ -46,7 +46,7 @@ public abstract class OldGeneration extends Generation {
     abstract void beginPromotion(boolean incrementalGc);
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor);
+    abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor);
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     abstract boolean scanGreyObjects(boolean incrementalGc);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java

@@ -24,6 +24,7 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static com.oracle.svm.core.heap.ReferenceInternals.getReferentFieldAddress;
 import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
 import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability;
 
@@ -124,7 +125,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) {
         Object refObject = referentAddr.toObjectNonNull();
         if (willSurviveThisCollection(refObject)) {
             // Either an object that got promoted without being moved or an object in the old gen.
-            RememberedSet.get().dirtyCardIfNecessary(dr, refObject);
+            RememberedSet.get().dirtyCardIfNecessary(dr, refObject, getReferentFieldAddress(dr));
             return;
         }
         if (!softReferencesAreWeak && dr instanceof SoftReference) {
@@ -216,7 +217,7 @@ private static boolean processRememberedRef(Reference<?> dr) {
         }
         Object refObject = refPointer.toObjectNonNull();
         if (willSurviveThisCollection(refObject)) {
-            RememberedSet.get().dirtyCardIfNecessary(dr, refObject);
+            RememberedSet.get().dirtyCardIfNecessary(dr, refObject, getReferentFieldAddress(dr));
             return true;
         }
         /*