Skip to content

Commit f5d2e59

Browse files
christianhaeublchristianhaeubl
committed
[GR-44099] Optimize card table scanning.
PullRequest: graal/13778
2 parents 7b63dd0 + 48d2f66 commit f5d2e59

File tree

3 files changed

+76
-41
lines changed

3 files changed

+76
-41
lines changed

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java‎

Lines changed: 57 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ final class AlignedChunkRememberedSet {
5454
private AlignedChunkRememberedSet() {
5555
}
5656

57+
@Fold
58+
public static int wordSize() {
59+
return ConfigurationValues.getTarget().wordSize;
60+
}
61+
5762
@Fold
5863
public static UnsignedWord getHeaderSize() {
5964
UnsignedWord headerSize = getFirstObjectTableLimitOffset();
@@ -126,28 +131,67 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) {
126131
}
127132

128133
public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
129-
Pointer cardTableStart = getCardTableStart(chunk);
130-
Pointer fotStart = getFirstObjectTableStart(chunk);
131134
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
132135
Pointer objectsLimit = HeapChunk.getTopPointer(chunk);
133136
UnsignedWord memorySize = objectsLimit.subtract(objectsStart);
134-
UnsignedWord indexLimit = CardTable.indexLimitForMemorySize(memorySize);
135137

136-
for (UnsignedWord index = WordFactory.zero(); index.belowThan(indexLimit); index = index.add(1)) {
137-
if (CardTable.isDirty(cardTableStart, index)) {
138+
Pointer cardTableStart = getCardTableStart(chunk);
139+
Pointer cardTableLimit = cardTableStart.add(CardTable.tableSizeForMemorySize(memorySize));
140+
141+
assert cardTableStart.unsignedRemainder(wordSize()).equal(0);
142+
assert getCardTableSize().unsignedRemainder(wordSize()).equal(0);
143+
144+
Pointer dirtyHeapStart = objectsLimit;
145+
Pointer dirtyHeapEnd = objectsLimit;
146+
Pointer cardPos = cardTableLimit.subtract(1);
147+
Pointer heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart);
148+
149+
while (cardPos.aboveOrEqual(cardTableStart)) {
150+
if (cardPos.readByte(0) != CardTable.CLEAN_ENTRY) {
138151
if (clean) {
139-
CardTable.setClean(cardTableStart, index);
152+
cardPos.writeByte(0, CardTable.CLEAN_ENTRY);
153+
}
154+
dirtyHeapStart = heapPos;
155+
} else {
156+
/* Hit a clean card, so process the dirty range. */
157+
if (dirtyHeapStart.belowThan(dirtyHeapEnd)) {
158+
walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor);
140159
}
141160

142-
Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, objectsLimit, index);
143-
Pointer cardLimit = CardTable.indexToMemoryPointer(objectsStart, index.add(1));
144-
Pointer walkLimit = PointerUtils.min(cardLimit, objectsLimit);
145-
while (ptr.belowThan(walkLimit)) {
146-
Object obj = ptr.toObject();
147-
visitor.visitObjectInline(obj);
148-
ptr = LayoutEncoding.getObjectEndInlineInGC(obj);
161+
if (PointerUtils.isAMultiple(cardPos, WordFactory.unsigned(wordSize()))) {
162+
/* Fast forward through word-aligned continuous range of clean cards. */
163+
cardPos = cardPos.subtract(wordSize());
164+
while (cardPos.aboveOrEqual(cardTableStart) && ((UnsignedWord) cardPos.readWord(0)).equal(CardTable.CLEAN_WORD)) {
165+
cardPos = cardPos.subtract(wordSize());
166+
}
167+
cardPos = cardPos.add(wordSize());
168+
heapPos = CardTable.cardToHeapAddress(cardTableStart, cardPos, objectsStart);
149169
}
170+
171+
/* Reset the dirty range. */
172+
dirtyHeapEnd = heapPos;
173+
dirtyHeapStart = heapPos;
150174
}
175+
176+
cardPos = cardPos.subtract(1);
177+
heapPos = heapPos.subtract(CardTable.BYTES_COVERED_BY_ENTRY);
178+
}
179+
180+
/* Process the remaining dirty range. */
181+
if (dirtyHeapStart.belowThan(dirtyHeapEnd)) {
182+
walkObjects(chunk, dirtyHeapStart, dirtyHeapEnd, visitor);
183+
}
184+
}
185+
186+
private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) {
187+
Pointer fotStart = getFirstObjectTableStart(chunk);
188+
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
189+
UnsignedWord index = CardTable.memoryOffsetToIndex(start.subtract(objectsStart));
190+
Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, index);
191+
while (ptr.belowThan(end)) {
192+
Object obj = ptr.toObject();
193+
visitor.visitObjectInline(obj);
194+
ptr = LayoutEncoding.getObjectEndInlineInGC(obj);
151195
}
152196
}
153197

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -74,32 +74,30 @@
7474
final class CardTable {
7575
public static final int BYTES_COVERED_BY_ENTRY = 512;
7676

77-
private static final int ENTRY_SIZE_BYTES = 1;
78-
79-
private static final int DIRTY_ENTRY = 0;
80-
private static final int CLEAN_ENTRY = 1;
77+
static final byte DIRTY_ENTRY = 0;
78+
static final byte CLEAN_ENTRY = 1;
79+
static final UnsignedWord CLEAN_WORD = WordFactory.unsigned(0x0101010101010101L);
8180

8281
private static final CardTableVerificationVisitor CARD_TABLE_VERIFICATION_VISITOR = new CardTableVerificationVisitor();
8382

8483
private CardTable() {
8584
}
8685

8786
public static void cleanTable(Pointer tableStart, UnsignedWord size) {
88-
UnmanagedMemoryUtil.fill(tableStart, size, (byte) CLEAN_ENTRY);
87+
UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY);
8988
}
9089

9190
public static void setDirty(Pointer table, UnsignedWord index) {
92-
UnsignedWord tableOffset = indexToTableOffset(index);
93-
byte valueBefore = table.readByte(tableOffset, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
91+
byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
9492
// Using a likely probability should typically avoid placing the write below at a separate
9593
// location with an extra jump back to after the barrier for more compact code.
9694
if (BranchProbabilityNode.probability(BranchProbabilityNode.LIKELY_PROBABILITY, valueBefore != DIRTY_ENTRY)) {
97-
table.writeByte(tableOffset, (byte) DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
95+
table.writeByte(index, DIRTY_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
9896
}
9997
}
10098

10199
public static void setClean(Pointer table, UnsignedWord index) {
102-
table.writeByte(indexToTableOffset(index), (byte) CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
100+
table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION);
103101
}
104102

105103
public static boolean isDirty(Pointer table, UnsignedWord index) {
@@ -113,25 +111,20 @@ private static boolean isClean(Pointer table, UnsignedWord index) {
113111
}
114112

115113
private static int readEntry(Pointer table, UnsignedWord index) {
116-
return table.readByte(indexToTableOffset(index));
117-
}
118-
119-
private static UnsignedWord indexToTableOffset(UnsignedWord index) {
120-
return index.multiply(ENTRY_SIZE_BYTES);
114+
return table.readByte(index);
121115
}
122116

123117
public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) {
124118
return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY);
125119
}
126120

127-
public static Pointer indexToMemoryPointer(Pointer memoryStart, UnsignedWord index) {
128-
UnsignedWord offset = index.multiply(BYTES_COVERED_BY_ENTRY);
129-
return memoryStart.add(offset);
121+
public static Pointer cardToHeapAddress(Pointer cardTableStart, Pointer cardAddr, Pointer objectsStart) {
122+
UnsignedWord offset = cardAddr.subtract(cardTableStart).multiply(CardTable.BYTES_COVERED_BY_ENTRY);
123+
return objectsStart.add(offset);
130124
}
131125

132126
public static UnsignedWord tableSizeForMemorySize(UnsignedWord memorySize) {
133-
UnsignedWord maxIndex = indexLimitForMemorySize(memorySize);
134-
return maxIndex.multiply(ENTRY_SIZE_BYTES);
127+
return indexLimitForMemorySize(memorySize);
135128
}
136129

137130
public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) {
@@ -178,7 +171,7 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta
178171
boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject);
179172
if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) {
180173
UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart));
181-
Pointer cardTableAddress = cardTableStart.add(indexToTableOffset(cardTableIndex));
174+
Pointer cardTableAddress = cardTableStart.add(cardTableIndex);
182175
Log.log().string("Object ").zhex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')')
183176
.string(fromImageHeap ? ", which is in the image heap, " : " ")
184177
.string("has an object reference at ")

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge.remset;
2626

27-
import com.oracle.svm.core.AlwaysInline;
2827
import org.graalvm.word.Pointer;
2928
import org.graalvm.word.UnsignedWord;
3029
import org.graalvm.word.WordFactory;
3130

32-
import com.oracle.svm.core.UnmanagedMemoryUtil;
31+
import com.oracle.svm.core.AlwaysInline;
3332
import com.oracle.svm.core.SubstrateUtil;
33+
import com.oracle.svm.core.UnmanagedMemoryUtil;
3434
import com.oracle.svm.core.config.ConfigurationValues;
3535
import com.oracle.svm.core.hub.LayoutEncoding;
3636
import com.oracle.svm.core.log.Log;
@@ -232,9 +232,9 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un
232232
* outside the current card.
233233
*/
234234
@AlwaysInline("GC performance")
235-
public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) {
235+
public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
236236
Pointer result;
237-
Pointer firstObject = getFirstObject(tableStart, objectsStart, objectsLimit, index);
237+
Pointer firstObject = getFirstObject(tableStart, objectsStart, index);
238238
Pointer indexedMemoryStart = objectsStart.add(indexToMemoryOffset(index));
239239
// If the object starts before the memory for this index, skip over it.
240240
if (firstObject.belowThan(indexedMemoryStart)) {
@@ -245,12 +245,11 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object
245245
result = indexedMemoryStart;
246246
}
247247
assert objectsStart.belowOrEqual(result) : "memoryStart.belowOrEqual(result)";
248-
assert result.belowOrEqual(objectsLimit) : "result.belowOrEqual(memoryLimit)";
249248
return result;
250249
}
251250

252251
@AlwaysInline("GC performance")
253-
private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit, UnsignedWord index) {
252+
private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
254253
UnsignedWord currentIndex = index;
255254
int currentEntry = getEntryAtIndex(tableStart, currentIndex);
256255
assert currentEntry != UNINITIALIZED_ENTRY : "uninitialized first object table entry";
@@ -276,7 +275,6 @@ private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart,
276275
UnsignedWord memoryOffset = entryToMemoryOffset(currentIndex, currentEntry);
277276
Pointer result = objectsStart.add(memoryOffset);
278277
assert objectsStart.belowOrEqual(result) : "chunkStart.belowOrEqual(result)";
279-
assert result.belowThan(objectsLimit) : "result.belowThan(memoryLimit)";
280278
return result;
281279
}
282280

@@ -292,7 +290,7 @@ private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) {
292290
public static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit) {
293291
UnsignedWord indexLimit = getTableSizeForMemoryRange(objectsStart, objectsLimit);
294292
for (UnsignedWord index = WordFactory.unsigned(0); index.belowThan(indexLimit); index = index.add(1)) {
295-
Pointer objStart = getFirstObject(tableStart, objectsStart, objectsLimit, index);
293+
Pointer objStart = getFirstObject(tableStart, objectsStart, index);
296294
if (objStart.belowThan(objectsStart) || objectsLimit.belowOrEqual(objStart)) {
297295
Log.log().string("The first object table entry at index ").unsigned(index).string(" points to an object that is outside of the current chunk: obj: ").zhex(objStart)
298296
.string(", chunk: ")

0 commit comments

Comments
 (0)