Skip to content

[GR-44099] Optimize card table scanning. #5963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 21, 2023
Merged

[GR-44099] Optimize card table scanning. #5963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cd88d38
Word-by-word card table scanning
petermz Jan 31, 2023
df9443f
Merge with master.
christianhaeubl Feb 16, 2023
48d2f66
Optimize card table scanning.
christianhaeubl Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 57 additions & 13 deletions ...ore.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ final class AlignedChunkRememberedSet {
private AlignedChunkRememberedSet() {
}

@Fold
public static int wordSize() {
return ConfigurationValues.getTarget().wordSize;
}

@Fold
public static UnsignedWord getHeaderSize() {
UnsignedWord headerSize = getFirstObjectTableLimitOffset();
Expand Down Expand Up @@ -126,28 +131,67 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) {
}

public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
Pointer cardTableStart = getCardTableStart(chunk);
Pointer fotStart = getFirstObjectTableStart(chunk);
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
Pointer objectsLimit = HeapChunk.getTopPointer(chunk);
UnsignedWord memorySize = objectsLimit.subtract(objectsStart);
UnsignedWord indexLimit = CardTable.indexLimitForMemorySize(memorySize);

for (UnsignedWord index = WordFactory.zero(); index.belowThan(indexLimit); index = index.add(1)) {
if (CardTable.isDirty(cardTableStart, index)) {
Pointer cardTableStart = getCardTableStart(chunk);
Pointer cardTableLimit = cardTableStart.add(CardTable.tableSizeForMemorySize(memorySize));

assert cardTableStart.unsignedRemainder(wordSize()).equal(0);
assert getCardTableSize().unsignedRemainder(wordSize()).equal(0);

Pointer dirtyHeapStart = objectsLimit;
Pointer dirtyHeapEnd = objectsLimit;
Pointer cardPos = cardTableLimit.subtract(1);
Pointer heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart);

while (cardPos.aboveOrEqual(cardTableStart)) {
if (cardPos.readByte(0) != CardTable.CLEAN_ENTRY) {
if (clean) {
CardTable.setClean(cardTableStart, index);
cardPos.writeByte(0, CardTable.CLEAN_ENTRY);
}
dirtyHeapStart = heapPos;
} else {
/* Hit a clean card, so process the dirty range. */
if (dirtyHeapStart.belowThan(dirtyHeapEnd)) {
walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor);
}

Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, objectsLimit, index);
Pointer cardLimit = CardTable.indexToMemoryPointer(objectsStart, index.add(1));
Pointer walkLimit = PointerUtils.min(cardLimit, objectsLimit);
while (ptr.belowThan(walkLimit)) {
Object obj = ptr.toObject();
visitor.visitObjectInline(obj);
ptr = LayoutEncoding.getObjectEndInlineInGC(obj);
if (PointerUtils.isAMultiple(cardPos, WordFactory.unsigned(wordSize()))) {
/* Fast forward through word-aligned continuous range of clean cards. */
cardPos = cardPos.subtract(wordSize());
while (cardPos.aboveOrEqual(cardTableStart) && ((UnsignedWord) cardPos.readWord(0)).equal(CardTable.CLEAN_WORD)) {
cardPos = cardPos.subtract(wordSize());
}
Copy link
Contributor

@petermz petermz Feb 14, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A do-while maybe?

Copy link
Member

@christianhaeubl christianhaeubl Feb 15, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A do-while loop would be incorrect because there is no guarantee that the whole word is clean.

cardPos = cardPos.add(wordSize());
heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart);
}

/* Reset the dirty range. */
dirtyHeapEnd = heapPos;
dirtyHeapStart = heapPos;
}

cardPos = cardPos.subtract(1);
heapPos = heapPos.subtract(CardTable.BYTES_COVERED_BY_ENTRY);
}

/* Process the remaining dirty range. */
if (dirtyHeapStart.belowThan(dirtyHeapEnd)) {
walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor);
}
}

private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) {
Pointer fotStart = getFirstObjectTableStart(chunk);
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
UnsignedWord index = CardTable.memoryOffsetToIndex(start.subtract(objectsStart));
Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, index);
while (ptr.belowThan(end)) {
Object obj = ptr.toObject();
visitor.visitObjectInline(obj);
ptr = LayoutEncoding.getObjectEndInlineInGC(obj);
}
}

Expand Down
33 changes: 13 additions & 20 deletions ...com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,32 +74,30 @@
final class CardTable {
public static final int BYTES_COVERED_BY_ENTRY = 512;

private static final int ENTRY_SIZE_BYTES = 1;

private static final int DIRTY_ENTRY = 0;
private static final int CLEAN_ENTRY = 1;
static final byte DIRTY_ENTRY = 0;
static final byte CLEAN_ENTRY = 1;
static final UnsignedWord CLEAN_WORD = WordFactory.unsigned(0x0101010101010101L);

private static final CardTableVerificationVisitor CARD_TABLE_VERIFICATION_VISITOR = new CardTableVerificationVisitor();

private CardTable() {
}

public static void cleanTable(Pointer tableStart, UnsignedWord size) {
UnmanagedMemoryUtil.fill(tableStart, size, (byte) CLEAN_ENTRY);
UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY);
}

public static void setDirty(Pointer table, UnsignedWord index) {
UnsignedWord tableOffset = indexToTableOffset(index);
byte valueBefore = table.readByte(tableOffset, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
// Using a likely probability should typically avoid placing the write below at a separate
// location with an extra jump back to after the barrier for more compact code.
if (BranchProbabilityNode.probability(BranchProbabilityNode.LIKELY_PROBABILITY, valueBefore != DIRTY_ENTRY)) {
table.writeByte(tableOffset, (byte) DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
table.writeByte(index, DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
}
}

public static void setClean(Pointer table, UnsignedWord index) {
table.writeByte(indexToTableOffset(index), (byte) CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
}

public static boolean isDirty(Pointer table, UnsignedWord index) {
Expand All @@ -113,25 +111,20 @@ private static boolean isClean(Pointer table, UnsignedWord index) {
}

private static int readEntry(Pointer table, UnsignedWord index) {
return table.readByte(indexToTableOffset(index));
}

private static UnsignedWord indexToTableOffset(UnsignedWord index) {
return index.multiply(ENTRY_SIZE_BYTES);
return table.readByte(index);
}

public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) {
return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY);
}

public static Pointer indexToMemoryPointer(Pointer memoryStart, UnsignedWord index) {
UnsignedWord offset = index.multiply(BYTES_COVERED_BY_ENTRY);
return memoryStart.add(offset);
public static Pointer cardToHeapAddress(Pointer cardTableStart, Pointer cardAddr, Pointer objectsStart) {
UnsignedWord offset = cardAddr.subtract(cardTableStart).multiply(CardTable.BYTES_COVERED_BY_ENTRY);
return objectsStart.add(offset);
}

public static UnsignedWord tableSizeForMemorySize(UnsignedWord memorySize) {
UnsignedWord maxIndex = indexLimitForMemorySize(memorySize);
return maxIndex.multiply(ENTRY_SIZE_BYTES);
return indexLimitForMemorySize(memorySize);
}

public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) {
Expand Down Expand Up @@ -178,7 +171,7 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta
boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject);
if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) {
UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart));
Pointer cardTableAddress = cardTableStart.add(indexToTableOffset(cardTableIndex));
Pointer cardTableAddress = cardTableStart.add(cardTableIndex);
Log.log().string("Object ").zhex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')')
.string(fromImageHeap ? ", which is in the image heap, " : " ")
.string("has an object reference at ")
Expand Down
14 changes: 6 additions & 8 deletions ...cle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@
*/
package com.oracle.svm.core.genscavenge.remset;

import com.oracle.svm.core.AlwaysInline;
import org.graalvm.word.Pointer;
import org.graalvm.word.UnsignedWord;
import org.graalvm.word.WordFactory;

import com.oracle.svm.core.UnmanagedMemoryUtil;
import com.oracle.svm.core.AlwaysInline;
import com.oracle.svm.core.SubstrateUtil;
import com.oracle.svm.core.UnmanagedMemoryUtil;
import com.oracle.svm.core.config.ConfigurationValues;
import com.oracle.svm.core.hub.LayoutEncoding;
import com.oracle.svm.core.log.Log;
Expand Down Expand Up @@ -232,9 +232,9 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un
* outside the current card.
*/
@AlwaysInline("GC performance")
public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) {
public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
Pointer result;
Pointer firstObject = getFirstObject(tableStart, objectsStart, objectsLimit, index);
Pointer firstObject = getFirstObject(tableStart, objectsStart, index);
Pointer indexedMemoryStart = objectsStart.add(indexToMemoryOffset(index));
// If the object starts before the memory for this index, skip over it.
if (firstObject.belowThan(indexedMemoryStart)) {
Expand All @@ -245,12 +245,11 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object
result = indexedMemoryStart;
}
assert objectsStart.belowOrEqual(result) : "memoryStart.belowOrEqual(result)";
assert result.belowOrEqual(objectsLimit) : "result.belowOrEqual(memoryLimit)";
return result;
}

@AlwaysInline("GC performance")
private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) {
private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
UnsignedWord currentIndex = index;
int currentEntry = getEntryAtIndex(tableStart, currentIndex);
assert currentEntry != UNINITIALIZED_ENTRY : "uninitialized first object table entry";
Expand All @@ -276,7 +275,6 @@ private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart,
UnsignedWord memoryOffset = entryToMemoryOffset(currentIndex, currentEntry);
Pointer result = objectsStart.add(memoryOffset);
assert objectsStart.belowOrEqual(result) : "chunkStart.belowOrEqual(result)";
assert result.belowThan(objectsLimit) : "result.belowThan(memoryLimit)";
return result;
}

Expand All @@ -292,7 +290,7 @@ private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) {
public static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit) {
UnsignedWord indexLimit = getTableSizeForMemoryRange(objectsStart, objectsLimit);
for (UnsignedWord index = WordFactory.unsigned(0); index.belowThan(indexLimit); index = index.add(1)) {
Pointer objStart = getFirstObject(tableStart, objectsStart, objectsLimit, index);
Pointer objStart = getFirstObject(tableStart, objectsStart, index);
if (objStart.belowThan(objectsStart) || objectsLimit.belowOrEqual(objStart)) {
Log.log().string("The first object table entry at index ").unsigned(index).string(" points to an object that is outside of the current chunk: obj: ").zhex(objStart)
.string(", chunk: ")
Expand Down