diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index ed25afb9808d..d244d47c7cb3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -54,6 +54,11 @@ final class AlignedChunkRememberedSet { private AlignedChunkRememberedSet() { } + @Fold + public static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + @Fold public static UnsignedWord getHeaderSize() { UnsignedWord headerSize = getFirstObjectTableLimitOffset(); @@ -126,28 +131,67 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) { } public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { - Pointer cardTableStart = getCardTableStart(chunk); - Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); Pointer objectsLimit = HeapChunk.getTopPointer(chunk); UnsignedWord memorySize = objectsLimit.subtract(objectsStart); - UnsignedWord indexLimit = CardTable.indexLimitForMemorySize(memorySize); - for (UnsignedWord index = WordFactory.zero(); index.belowThan(indexLimit); index = index.add(1)) { - if (CardTable.isDirty(cardTableStart, index)) { + Pointer cardTableStart = getCardTableStart(chunk); + Pointer cardTableLimit = cardTableStart.add(CardTable.tableSizeForMemorySize(memorySize)); + + assert cardTableStart.unsignedRemainder(wordSize()).equal(0); + assert getCardTableSize().unsignedRemainder(wordSize()).equal(0); + + Pointer dirtyHeapStart = objectsLimit; + Pointer dirtyHeapEnd = objectsLimit; + Pointer cardPos = cardTableLimit.subtract(1); + Pointer heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart); + + while (cardPos.aboveOrEqual(cardTableStart)) { + if (cardPos.readByte(0) != CardTable.CLEAN_ENTRY) { if (clean) { - CardTable.setClean(cardTableStart, index); + cardPos.writeByte(0, CardTable.CLEAN_ENTRY); + } + dirtyHeapStart = heapPos; + } else { + /* Hit a clean card, so process the dirty range. */ + if (dirtyHeapStart.belowThan(dirtyHeapEnd)) { + walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor); } - Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, objectsLimit, index); - Pointer cardLimit = CardTable.indexToMemoryPointer(objectsStart, index.add(1)); - Pointer walkLimit = PointerUtils.min(cardLimit, objectsLimit); - while (ptr.belowThan(walkLimit)) { - Object obj = ptr.toObject(); - visitor.visitObjectInline(obj); - ptr = LayoutEncoding.getObjectEndInlineInGC(obj); + if (PointerUtils.isAMultiple(cardPos, WordFactory.unsigned(wordSize()))) { + /* Fast forward through word-aligned continuous range of clean cards. */ + cardPos = cardPos.subtract(wordSize()); + while (cardPos.aboveOrEqual(cardTableStart) && ((UnsignedWord) cardPos.readWord(0)).equal(CardTable.CLEAN_WORD)) { + cardPos = cardPos.subtract(wordSize()); + } + cardPos = cardPos.add(wordSize()); + heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart); } + + /* Reset the dirty range. */ + dirtyHeapEnd = heapPos; + dirtyHeapStart = heapPos; } + + cardPos = cardPos.subtract(1); + heapPos = heapPos.subtract(CardTable.BYTES_COVERED_BY_ENTRY); + } + + /* Process the remaining dirty range. */ + if (dirtyHeapStart.belowThan(dirtyHeapEnd)) { + walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor); + } + } + + private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) { + Pointer fotStart = getFirstObjectTableStart(chunk); + Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); + UnsignedWord index = CardTable.memoryOffsetToIndex(start.subtract(objectsStart)); + Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, index); + while (ptr.belowThan(end)) { + Object obj = ptr.toObject(); + visitor.visitObjectInline(obj); + ptr = LayoutEncoding.getObjectEndInlineInGC(obj); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index 4796a9dde18f..fa707695db94 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -74,10 +74,9 @@ final class CardTable { public static final int BYTES_COVERED_BY_ENTRY = 512; - private static final int ENTRY_SIZE_BYTES = 1; - - private static final int DIRTY_ENTRY = 0; - private static final int CLEAN_ENTRY = 1; + static final byte DIRTY_ENTRY = 0; + static final byte CLEAN_ENTRY = 1; + static final UnsignedWord CLEAN_WORD = WordFactory.unsigned(0x0101010101010101L); private static final CardTableVerificationVisitor CARD_TABLE_VERIFICATION_VISITOR = new CardTableVerificationVisitor(); @@ -85,21 +84,20 @@ private CardTable() { } public static void cleanTable(Pointer tableStart, UnsignedWord size) { - UnmanagedMemoryUtil.fill(tableStart, size, (byte) CLEAN_ENTRY); + UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } public static void setDirty(Pointer table, UnsignedWord index) { - UnsignedWord tableOffset = indexToTableOffset(index); - byte valueBefore = table.readByte(tableOffset, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); + byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate // location with an extra jump back to after the barrier for more compact code. if (BranchProbabilityNode.probability(BranchProbabilityNode.LIKELY_PROBABILITY, valueBefore != DIRTY_ENTRY)) { - table.writeByte(tableOffset, (byte) DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); + table.writeByte(index, DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } } public static void setClean(Pointer table, UnsignedWord index) { - table.writeByte(indexToTableOffset(index), (byte) CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); + table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } public static boolean isDirty(Pointer table, UnsignedWord index) { @@ -113,25 +111,20 @@ private static boolean isClean(Pointer table, UnsignedWord index) { } private static int readEntry(Pointer table, UnsignedWord index) { - return table.readByte(indexToTableOffset(index)); - } - - private static UnsignedWord indexToTableOffset(UnsignedWord index) { - return index.multiply(ENTRY_SIZE_BYTES); + return table.readByte(index); } public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } - public static Pointer indexToMemoryPointer(Pointer memoryStart, UnsignedWord index) { - UnsignedWord offset = index.multiply(BYTES_COVERED_BY_ENTRY); - return memoryStart.add(offset); + public static Pointer cardToHeapAddress(Pointer cardTableStart, Pointer cardAddr, Pointer objectsStart) { + UnsignedWord offset = cardAddr.subtract(cardTableStart).multiply(CardTable.BYTES_COVERED_BY_ENTRY); + return objectsStart.add(offset); } public static UnsignedWord tableSizeForMemorySize(UnsignedWord memorySize) { - UnsignedWord maxIndex = indexLimitForMemorySize(memorySize); - return maxIndex.multiply(ENTRY_SIZE_BYTES); + return indexLimitForMemorySize(memorySize); } public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) { @@ -178,7 +171,7 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject); if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) { UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart)); - Pointer cardTableAddress = cardTableStart.add(indexToTableOffset(cardTableIndex)); + Pointer cardTableAddress = cardTableStart.add(cardTableIndex); Log.log().string("Object ").zhex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')') .string(fromImageHeap ? ", which is in the image heap, " : " ") .string("has an object reference at ") diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 1183aa2712dc..2c5f3ff3b141 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -24,13 +24,13 @@ */ package com.oracle.svm.core.genscavenge.remset; -import com.oracle.svm.core.AlwaysInline; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; @@ -232,9 +232,9 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un * outside the current card. */ @AlwaysInline("GC performance") - public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) { + public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { Pointer result; - Pointer firstObject = getFirstObject(tableStart, objectsStart, objectsLimit, index); + Pointer firstObject = getFirstObject(tableStart, objectsStart, index); Pointer indexedMemoryStart = objectsStart.add(indexToMemoryOffset(index)); // If the object starts before the memory for this index, skip over it. if (firstObject.belowThan(indexedMemoryStart)) { @@ -245,12 +245,11 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object result = indexedMemoryStart; } assert objectsStart.belowOrEqual(result) : "memoryStart.belowOrEqual(result)"; - assert result.belowOrEqual(objectsLimit) : "result.belowOrEqual(memoryLimit)"; return result; } @AlwaysInline("GC performance") - private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) { + private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { UnsignedWord currentIndex = index; int currentEntry = getEntryAtIndex(tableStart, currentIndex); assert currentEntry != UNINITIALIZED_ENTRY : "uninitialized first object table entry"; @@ -276,7 +275,6 @@ private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord memoryOffset = entryToMemoryOffset(currentIndex, currentEntry); Pointer result = objectsStart.add(memoryOffset); assert objectsStart.belowOrEqual(result) : "chunkStart.belowOrEqual(result)"; - assert result.belowThan(objectsLimit) : "result.belowThan(memoryLimit)"; return result; } @@ -292,7 +290,7 @@ private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) { public static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit) { UnsignedWord indexLimit = getTableSizeForMemoryRange(objectsStart, objectsLimit); for (UnsignedWord index = WordFactory.unsigned(0); index.belowThan(indexLimit); index = index.add(1)) { - Pointer objStart = getFirstObject(tableStart, objectsStart, objectsLimit, index); + Pointer objStart = getFirstObject(tableStart, objectsStart, index); if (objStart.belowThan(objectsStart) || objectsLimit.belowOrEqual(objStart)) { Log.log().string("The first object table entry at index ").unsigned(index).string(" points to an object that is outside of the current chunk: obj: ").zhex(objStart) .string(", chunk: ")