Allocate TLAB as part of chunks and size them dynamically

Author: Thomas Schrott

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java

@@ -221,10 +221,7 @@ public static final class OnlyCompletely extends BasicPolicy {
 
         @Override
         public boolean shouldCollectCompletely(boolean followingIncrementalCollection) {
-            if (!followingIncrementalCollection && shouldCollectYoungGenSeparately(false)) {
-                return false;
-            }
-            return true;
+            return followingIncrementalCollection || !shouldCollectYoungGenSeparately(false);
         }
 
         @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -260,6 +260,7 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) ||
             collectionTimer.stop();
         }
 
+        resizeAllTlabs();
         accounting.updateCollectionCountAndTime(completeCollection, collectionTimer.totalNanos());
         HeapImpl.getAccounting().notifyAfterCollection();
         GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection();
@@ -402,6 +403,14 @@ public static UnsignedWord getChunkBytes() {
         return youngBytes.add(oldBytes);
     }
 
+    private static void resizeAllTlabs() {
+        if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) {
+            for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
+                TlabSupport.resize(thread);
+            }
+        }
+    }
+
     private void printGCBefore(GCCause cause) {
         if (!SubstrateGCOptions.VerboseGC.getValue()) {
             return;
@@ -1317,7 +1326,7 @@ protected PrintGCSummaryOperation() {
 
         @Override
         protected void operate() {
-            ThreadLocalAllocation.disableAndFlushForAllThreads();
+            HeapImpl.getHeapImpl().makeParseable();
 
             Log log = Log.log();
             log.string("GC summary").indent(true);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java

@@ -50,8 +50,6 @@
 import com.oracle.svm.core.annotate.Substitute;
 import com.oracle.svm.core.annotate.TargetClass;
 import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
-import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor;
-import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
@@ -196,7 +194,7 @@ public boolean isInPrimaryImageHeap(Pointer objPointer) {
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void suspendAllocation() {
-        ThreadLocalAllocation.suspendInCurrentThread();
+        TlabSupport.suspendAllocationInCurrentThread();
     }
 
     @Override
@@ -415,13 +413,14 @@ public void endSafepoint() {
     @Uninterruptible(reason = "Called during startup.")
     @Override
     public void attachThread(IsolateThread isolateThread) {
-        // nothing to do
+        TlabSupport.startupInitialization();
+        TlabSupport.initialize(isolateThread);
     }
 
     @Override
     @Uninterruptible(reason = "Thread is detaching and holds the THREAD_MUTEX.")
     public void detachThread(IsolateThread isolateThread) {
-        ThreadLocalAllocation.disableAndFlushForThread(isolateThread);
+        TlabSupport.disableAndFlushForThread(isolateThread);
     }
 
     @Fold
@@ -707,7 +706,7 @@ public void optionValueChanged(RuntimeOptionKey<?> key) {
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public long getThreadAllocatedMemory(IsolateThread thread) {
-        UnsignedWord allocatedBytes = ThreadLocalAllocation.allocatedBytes.getVolatile(thread);
+        UnsignedWord allocatedBytes = ThreadLocalAllocation.getAllocatedBytes(thread);
 
         /*
          * The current aligned chunk in the TLAB is only partially filled and therefore not yet
@@ -717,9 +716,8 @@ public long getThreadAllocatedMemory(IsolateThread thread) {
          * can cause that the memory in the current aligned TLAB chunk is counted twice.
          */
         ThreadLocalAllocation.Descriptor tlab = ThreadLocalAllocation.getTlab(thread);
-        AlignedHeader alignedTlab = tlab.getAlignedChunk();
+        Pointer start = tlab.getAlignedAllocationStart(SubstrateAllocationSnippets.TLAB_START_IDENTITY);
         Pointer top = tlab.getAllocationTop(SubstrateAllocationSnippets.TLAB_TOP_IDENTITY);
-        Pointer start = AlignedHeapChunk.getObjectsStart(alignedTlab);
 
         if (top.aboveThan(start)) {
             UnsignedWord usedTlabSize = top.subtract(start);
@@ -787,7 +785,7 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc
         if (AuxiliaryImageHeap.isPresent() && AuxiliaryImageHeap.singleton().containsObject(ptr)) {
             log.string("points into the auxiliary image heap");
             return true;
-        } else if (printTlabInfo(log, ptr, CurrentIsolate.getCurrentThread())) {
+        } else if (TlabSupport.printTlabInfo(log, ptr, CurrentIsolate.getCurrentThread())) {
             return true;
         }
 
@@ -805,7 +803,7 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc
              * If we are not at a safepoint, then it is unsafe to access thread locals of another
              * thread as the IsolateThread could be freed at any time.
              */
-            if (printTlabInfo(log, ptr)) {
+            if (TlabSupport.printTlabInfo(log, ptr)) {
                 return true;
             }
         }
@@ -824,53 +822,6 @@ boolean isInHeap(Pointer ptr) {
         return isInImageHeap(ptr) || youngGeneration.isInSpace(ptr) || oldGeneration.isInSpace(ptr);
     }
 
-    private static boolean printTlabInfo(Log log, Pointer ptr) {
-        for (IsolateThread thread = VMThreads.firstThreadUnsafe(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
-            if (printTlabInfo(log, ptr, thread)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private static boolean printTlabInfo(Log log, Pointer ptr, IsolateThread thread) {
-        ThreadLocalAllocation.Descriptor tlab = getTlabUnsafe(thread);
-        AlignedHeader aChunk = tlab.getAlignedChunk();
-        while (aChunk.isNonNull()) {
-            if (HeapChunk.asPointer(aChunk).belowOrEqual(ptr) && ptr.belowThan(HeapChunk.getEndPointer(aChunk))) {
-                /* top may be null for a thread's current aligned allocation chunk. */
-                boolean unusablePart = HeapChunk.getTopPointer(aChunk).isNonNull() && ptr.aboveOrEqual(HeapChunk.getTopPointer(aChunk));
-                printTlabChunkInfo(log, thread, aChunk, "aligned", unusablePart);
-                return true;
-            }
-            aChunk = HeapChunk.getNext(aChunk);
-        }
-
-        UnalignedHeader uChunk = tlab.getUnalignedChunk();
-        while (uChunk.isNonNull()) {
-            if (HeapChunk.asPointer(uChunk).belowOrEqual(ptr) && ptr.belowThan(HeapChunk.getEndPointer(uChunk))) {
-                boolean unusablePart = ptr.aboveOrEqual(HeapChunk.getTopPointer(uChunk));
-                printTlabChunkInfo(log, thread, uChunk, "unaligned", unusablePart);
-                return true;
-            }
-            uChunk = HeapChunk.getNext(uChunk);
-        }
-
-        return false;
-    }
-
-    private static void printTlabChunkInfo(Log log, IsolateThread thread, HeapChunk.Header<?> chunk, String chunkType, boolean unusablePart) {
-        String unusable = unusablePart ? "unusable part of " : "";
-        log.string("points into ").string(unusable).string(chunkType).string(" chunk ").zhex(chunk).spaces(1);
-        log.string("(TLAB of thread ").zhex(thread).string(")");
-    }
-
-    @Uninterruptible(reason = "This whole method is unsafe, so it is only uninterruptible to satisfy the checks.")
-    static Descriptor getTlabUnsafe(IsolateThread thread) {
-        assert SubstrateDiagnostics.isFatalErrorHandlingThread() : "can cause crashes, so it may only be used while printing diagnostics";
-        return ThreadLocalAllocation.getTlab(thread);
-    }
-
     @Override
     @Uninterruptible(reason = "Called during early startup.")
     public boolean verifyImageHeapMapping() {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -40,6 +40,7 @@
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
 import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectVisitor;
@@ -549,7 +550,7 @@ private static boolean areEdenBytesCorrect() {
             /* Verify that there are no threads that have a TLAB. */
             for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
                 ThreadLocalAllocation.Descriptor tlab = ThreadLocalAllocation.getTlab(thread);
-                if (tlab.getAlignedChunk().isNonNull() || tlab.getUnalignedChunk().isNonNull()) {
+                if (tlab.getAlignedAllocationStart(SubstrateAllocationSnippets.TLAB_START_IDENTITY).isNonNull() || tlab.getUnalignedChunk().isNonNull()) {
                     return false;
                 }
             }