Write first object table during planning.

Author: peter-hofer

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -395,8 +395,8 @@ private void compact(Timers timers) {
                  * chunk during compaction. The remembered set bits are already set after planning.
                  */
                 if (!AlignedHeapChunk.isEmpty(chunk)) {
-                    RememberedSet.get().enableRememberedSetForChunk(chunk);
-                } // empty chunks will be freed or reset before reuse, no need to reinitialize here
+                    RememberedSet.get().clearRememberedSet(chunk);
+                } // else: chunks will be freed or reset before reuse, no need to reinitialize here
 
                 chunk = HeapChunk.getNext(chunk);
             }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -449,7 +449,7 @@ private Object copyAlignedObject(Object originalObj) {
             if (SerialGCOptions.useCompactingOldGen() && GCImpl.getGCImpl().isCompleteCollection()) {
                 /*
                  * In a compacting complete collection, the remembered set bit is set already during
-                 * marking and the first object table is built later during compaction.
+                 * marking and the first object table is built during planning.
                  */
             } else {
                 /*

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java

@@ -35,7 +35,9 @@
 import com.oracle.svm.core.genscavenge.HeapChunk;
 import com.oracle.svm.core.genscavenge.ObjectHeaderImpl;
 import com.oracle.svm.core.genscavenge.Space;
+import com.oracle.svm.core.genscavenge.remset.AlignedChunkRememberedSet;
 import com.oracle.svm.core.genscavenge.remset.BrickTable;
+import com.oracle.svm.core.genscavenge.remset.FirstObjectTable;
 import com.oracle.svm.core.hub.LayoutEncoding;
 
 import jdk.graal.compiler.word.Word;
@@ -57,6 +59,9 @@ public PlanningVisitor() {
     public void init(Space space) {
         allocChunk = space.getFirstAlignedHeapChunk();
         allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
+        if (!allocChunk.getShouldSweepInsteadOfCompact()) {
+            FirstObjectTable.initializeTable(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), AlignedChunkRememberedSet.getFirstObjectTableSize());
+        }
     }
 
     @Override
@@ -70,7 +75,7 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
         UnsignedWord brickIndex = WordFactory.zero();
 
         /* Initialize the move info structure at the chunk's object start location. */
-        ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+        ObjectMoveInfo.setNewAddress(objSeq, objSeq);
         ObjectMoveInfo.setObjectSeqSize(objSeq, WordFactory.zero());
         ObjectMoveInfo.setNextObjectSeqOffset(objSeq, WordFactory.zero());
 
@@ -116,11 +121,43 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
                 }
 
                 objSeqSize = objSeqSize.add(objSize);
+                if (!sweeping) {
+                    if (allocPointer.add(objSeqSize).aboveThan(AlignedHeapChunk.getObjectsEnd(allocChunk))) {
+                        /* Out of space, move to the start of the next chunk. */
+                        allocChunk = HeapChunk.getNext(allocChunk);
+                        assert allocChunk.isNonNull();
+                        assert !allocChunk.getShouldSweepInsteadOfCompact();
+                        allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
+
+                        /*
+                         * TODO: we should reset the FOT entries we already wrote in the last chunk
+                         * (but they should not be accessed, not even by heap verification)
+                         */
+
+                        /* Visit previous objects in sequence again to write new FOT entries. */
+                        FirstObjectTable.initializeTable(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), AlignedChunkRememberedSet.getFirstObjectTableSize());
+                        Pointer q = objSeq;
+                        while (q.notEqual(p)) {
+                            UnsignedWord offset = q.subtract(objSeq);
+                            UnsignedWord size = LayoutEncoding.getSizeFromObjectInlineInGC(q.toObject());
+                            FirstObjectTable.setTableForObject(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), offset, offset.add(size));
+                            q = q.add(size);
+                        }
+                    }
+
+                    Pointer allocEndOffset = allocPointer.add(objSeqSize).subtract(AlignedHeapChunk.getObjectsStart(allocChunk));
+                    FirstObjectTable.setTableForObject(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), allocEndOffset.subtract(objSize), allocEndOffset);
+                }
+
             } else { // not marked, i.e. not alive and start of a gap of yet unknown size
                 if (objSeqSize.notEqual(0)) { // end of an object sequence
-                    Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
-                    ObjectMoveInfo.setNewAddress(objSeq, newAddress);
                     ObjectMoveInfo.setObjectSeqSize(objSeq, objSeqSize);
+                    if (sweeping) {
+                        ObjectMoveInfo.setNewAddress(objSeq, objSeq);
+                    } else {
+                        ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+                        allocPointer = allocPointer.add(objSeqSize); // ensured enough memory above
+                    }
 
                     objSeqSize = WordFactory.zero();
 
@@ -137,13 +174,17 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
         }
         assert gapSize.equal(0) || objSeqSize.equal(0);
 
-        /* A gap at the end of the chunk requires updating the chunk top. */
         if (gapSize.notEqual(0)) {
             chunk.setTopOffset(chunk.getTopOffset().subtract(gapSize));
+
         } else if (objSeqSize.notEqual(0)) {
-            Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
-            ObjectMoveInfo.setNewAddress(objSeq, newAddress);
             ObjectMoveInfo.setObjectSeqSize(objSeq, objSeqSize);
+            if (sweeping) {
+                ObjectMoveInfo.setNewAddress(objSeq, objSeq);
+            } else {
+                ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+                allocPointer = allocPointer.add(objSeqSize); // ensured enough memory above
+            }
         }
 
         if (sweeping) {
@@ -169,17 +210,4 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
 
         return true;
     }
-
-    private Pointer allocate(UnsignedWord size) {
-        Pointer p = allocPointer;
-        allocPointer = allocPointer.add(size);
-        if (allocPointer.aboveThan(AlignedHeapChunk.getObjectsEnd(allocChunk))) {
-            allocChunk = HeapChunk.getNext(allocChunk);
-            assert allocChunk.isNonNull();
-
-            p = AlignedHeapChunk.getObjectsStart(allocChunk);
-            allocPointer = p.add(size);
-        }
-        return p;
-    }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java

@@ -54,7 +54,7 @@
 import jdk.graal.compiler.replacements.nodes.AssertionNode;
 import jdk.graal.compiler.word.Word;
 
-final class AlignedChunkRememberedSet {
+public final class AlignedChunkRememberedSet {
     private AlignedChunkRememberedSet() {
     }
 
@@ -239,7 +239,7 @@ static UnsignedWord getCardTableSize() {
     }
 
     @Fold
-    static UnsignedWord getFirstObjectTableSize() {
+    public static UnsignedWord getFirstObjectTableSize() {
         return getCardTableSize();
     }
 
@@ -286,7 +286,7 @@ private static Pointer getCardTableStart(Pointer chunk) {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    private static Pointer getFirstObjectTableStart(AlignedHeader chunk) {
+    public static Pointer getFirstObjectTableStart(AlignedHeader chunk) {
         return getFirstObjectTableStart(HeapChunk.asPointer(chunk));
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java

@@ -109,7 +109,7 @@
  * <p>
  * Implementation note: Table entries are bytes but converted to and from ints with bounds checks.
  */
-final class FirstObjectTable {
+public final class FirstObjectTable {
     /**
      * The number of bytes of memory covered by an entry. Since the indexes into the CardTable are
      * used to index into the FirstObjectTable, these need to have the same value.