Fixes and improvements.

Author: peter-hofer

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java

@@ -224,15 +224,15 @@ private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) {
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     private static Object slowPathNewArray(Word objectHeader, int length, int fillStartOffset) {
-        return slowPathNewArrayImpl(objectHeader, length, fillStartOffset, null);
+        return slowPathNewArrayOrPodImpl(objectHeader, length, fillStartOffset, null);
     }
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     private static Object slowPathNewPodInstance(Word objectHeader, int arrayLength, int fillStartOffset, byte[] referenceMap) {
-        return slowPathNewArrayImpl(objectHeader, arrayLength, fillStartOffset, referenceMap);
+        return slowPathNewArrayOrPodImpl(objectHeader, arrayLength, fillStartOffset, referenceMap);
     }
 
-    private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object slowPathNewArrayOrPodImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
         /*
          * Avoid stack overflow errors while producing memory chunks, because that could leave the
          * heap in an inconsistent state.
@@ -256,7 +256,7 @@ private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fi
                 throw OutOfMemoryUtil.reportOutOfMemoryError(outOfMemoryError);
             }
 
-            Object result = slowPathNewArrayWithoutAllocating(hub, length, size, fillStartOffset, podReferenceMap);
+            Object result = slowPathNewArrayOrPodWithoutAllocating(hub, length, size, fillStartOffset, podReferenceMap);
             runSlowPathHooks();
             return result;
         } finally {
@@ -265,26 +265,26 @@ private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fi
     }
 
     @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
-    private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object slowPathNewArrayOrPodWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         DeoptTester.disableDeoptTesting();
         try {
-            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayWithoutAllocating", DynamicHub.toClass(hub).getName());
+            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayOrPodWithoutAllocating", DynamicHub.toClass(hub).getName());
             GCImpl.getGCImpl().maybeCollectOnAllocation();
 
             if (size.aboveOrEqual(HeapParameters.getLargeArrayThreshold())) {
                 /* Large arrays go into their own unaligned chunk. */
                 boolean needsZeroing = !HeapChunkProvider.areUnalignedChunksZeroed();
                 UnalignedHeapChunk.UnalignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceUnalignedChunk(size);
-                return allocateLargeArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing, podReferenceMap);
+                return allocateLargeArrayOrPodInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing, podReferenceMap);
             }
             /* Small arrays go into the regular aligned chunk. */
 
             // We might have allocated in the caller and acquired a TLAB with enough space already
             // (but we need to check in an uninterruptible method to be safe)
-            Object array = allocateSmallArrayInCurrentTlab(hub, length, size, fillStartOffset, podReferenceMap);
+            Object array = allocateSmallArrayOrPodInCurrentTlab(hub, length, size, fillStartOffset, podReferenceMap);
             if (array == null) { // We need a new chunk.
                 AlignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceAlignedChunk();
-                array = allocateSmallArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, podReferenceMap);
+                array = allocateSmallArrayOrPodInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, podReferenceMap);
             }
             return array;
         } finally {
@@ -300,7 +300,7 @@ private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader ne
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object allocateSmallArrayOrPodInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         if (size.aboveThan(availableTlabMemory(getTlab()))) {
             return null;
         }
@@ -309,13 +309,15 @@ private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk, byte[] podReferenceMap) {
+    private static Object allocateSmallArrayOrPodInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk, byte[] podReferenceMap) {
         Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk);
         return formatArrayOrPod(memory, hub, length, false, FillContent.WITH_ZEROES, fillStartOffset, podReferenceMap);
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory, modifies TLAB")
-    private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, UnalignedHeader newTlabChunk, boolean needsZeroing, byte[] podReferenceMap) {
+    private static Object allocateLargeArrayOrPodInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset,
+                    UnalignedHeader newTlabChunk, boolean needsZeroing, byte[] podReferenceMap) {
+
         ThreadLocalAllocation.Descriptor tlab = getTlab();
 
         HeapChunk.setNext(newTlabChunk, tlab.getUnalignedChunk());
@@ -344,7 +346,7 @@ private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, Un
     private static Object formatArrayOrPod(Pointer memory, DynamicHub hub, int length, boolean unaligned, FillContent fillContent, int fillStartOffset, byte[] podReferenceMap) {
         Class<?> clazz = DynamicHub.toClass(hub);
         if (podReferenceMap != null) {
-            return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillStartOffset, true);
+            return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillContent, fillStartOffset, true);
         }
         return FormatArrayNode.formatArray(memory, clazz, length, false, unaligned, fillContent, fillStartOffset, true);
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java

@@ -135,7 +135,7 @@ public Object allocatePod(@NonNullParameter DynamicHub hub, int arrayLength, byt
         if (useTLAB() && probability(FAST_PATH_PROBABILITY, shouldAllocateInTLAB(allocationSize, true)) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
             writeTlabTop(thread, newTop);
             emitPrefetchAllocate(newTop, true);
-            instance = formatPod(top, hub, arrayLength, referenceMap, false, false, afterArrayLengthOffset(),
+            instance = formatPod(top, hub, arrayLength, referenceMap, false, false, FillContent.WITH_ZEROES, afterArrayLengthOffset(),
                             emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, profilingData.snippetCounters);
         } else {
             profilingData.snippetCounters.stub.inc();
@@ -146,23 +146,24 @@ public Object allocatePod(@NonNullParameter DynamicHub hub, int arrayLength, byt
     }
 
     @Snippet
-    public Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier,
-                    @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
+    public Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned,
+                    FillContent fillContents, int fillStartOffset, @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter boolean supportsBulkZeroing,
+                    @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
 
         DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
         byte[] refMapNonNull = (byte[]) PiNode.piCastNonNull(referenceMap, SnippetAnchorNode.anchor());
-        return formatPod(memory, hubNonNull, arrayLength, refMapNonNull, rememberedSet, unaligned, fillStartOffset,
+        return formatPod(memory, hubNonNull, arrayLength, refMapNonNull, rememberedSet, unaligned, fillContents, fillStartOffset,
                         emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
     }
 
-    private Object formatPod(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset,
+    private Object formatPod(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, FillContent fillContents, int fillStartOffset,
                     boolean emitMemoryBarrier, boolean maybeUnroll, boolean supportsBulkZeroing, boolean supportsOptimizedFilling, AllocationSnippetCounters snippetCounters) {
 
         int layoutEncoding = hub.getLayoutEncoding();
         UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, arrayLength);
 
         Word objectHeader = encodeAsObjectHeader(hub, rememberedSet, unaligned);
-        Object instance = formatArray(objectHeader, allocationSize, arrayLength, memory, FillContent.WITH_ZEROES, fillStartOffset,
+        Object instance = formatArray(objectHeader, allocationSize, arrayLength, memory, fillContents, fillStartOffset,
                         false, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
 
         int fromOffset = ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte);
@@ -341,8 +342,9 @@ public void lower(FormatPodNode node, LoweringTool tool) {
                 args.add("referenceMap", node.getReferenceMap());
                 args.add("rememberedSet", node.getRememberedSet());
                 args.add("unaligned", node.getUnaligned());
+                args.add("fillContents", node.getFillContents());
                 args.add("fillStartOffset", node.getFillStartOffset());
-                args.add("emitMemoryBarrier", node.getEmitMemoryBarrier());
+                args.addConst("emitMemoryBarrier", node.getEmitMemoryBarrier());
                 args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
                 args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
                 args.addConst("snippetCounters", snippetCounters);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java

@@ -33,6 +33,7 @@
 import org.graalvm.compiler.nodes.FixedWithNextNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.replacements.AllocationSnippets;
 import org.graalvm.word.Pointer;
 
 @NodeInfo(cycles = CYCLES_64, size = SIZE_64)
@@ -45,18 +46,20 @@ public class FormatPodNode extends FixedWithNextNode implements Lowerable {
     @Input protected ValueNode referenceMap;
     @Input protected ValueNode rememberedSet;
     @Input protected ValueNode unaligned;
+    @Input protected ValueNode fillContents;
     @Input protected ValueNode fillStartOffset;
-    @Input protected ValueNode emitMemoryBarrier;
+    private final boolean emitMemoryBarrier;
 
-    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap,
-                    ValueNode rememberedSet, ValueNode unaligned, ValueNode fillStartOffset, ValueNode emitMemoryBarrier) {
+    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap, ValueNode rememberedSet,
+                    ValueNode unaligned, ValueNode fillContents, ValueNode fillStartOffset, boolean emitMemoryBarrier) {
         super(TYPE, StampFactory.objectNonNull());
         this.memory = memory;
         this.hub = hub;
         this.arrayLength = arrayLength;
         this.referenceMap = referenceMap;
         this.rememberedSet = rememberedSet;
         this.unaligned = unaligned;
+        this.fillContents = fillContents;
         this.fillStartOffset = fillStartOffset;
         this.emitMemoryBarrier = emitMemoryBarrier;
     }
@@ -85,15 +88,19 @@ public ValueNode getUnaligned() {
         return unaligned;
     }
 
+    public ValueNode getFillContents() {
+        return fillContents;
+    }
+
     public ValueNode getFillStartOffset() {
         return fillStartOffset;
     }
 
-    public ValueNode getEmitMemoryBarrier() {
+    public boolean getEmitMemoryBarrier() {
         return emitMemoryBarrier;
     }
 
     @NodeIntrinsic
-    public static native Object formatPod(Pointer memory, Class<?> hub, int arrayLength, byte[] referenceMap,
-                    boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier);
+    public static native Object formatPod(Pointer memory, Class<?> hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned,
+                    AllocationSnippets.FillContent fillContents, int fillStartOffset, @ConstantNodeParameter boolean emitMemoryBarrier);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java

@@ -62,18 +62,18 @@
  * important to keep in mind because methods such as {@link Class#isInstance} will return
  * {@code true} and {@link Class#isArray()} will return {@code false}, while
  * {@link LayoutEncoding#isPureInstance} will return {@code false} and
- * {@link LayoutEncoding#isArrayLike} will return {@code true} and for hybrid objects.
+ * {@link LayoutEncoding#isArrayLike} will return {@code true} for hybrid objects.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface Hybrid {
 
     /**
-     * The type of the array part of the hybrid class. Must be specified if no field annotated
-     * with @{@link Hybrid.Array} is declared, otherwise that field's type determines the type of
-     * the array part.
+     * The component type of the array part of the hybrid class. Must be specified if no field
+     * annotated with @{@link Hybrid.Array} is declared, otherwise that field's type determines the
+     * type of the array part.
      */
-    Class<?> arrayType() default Array.class;
+    Class<?> componentType() default void.class;
 
     /**
      * If {@code true}, allow the data in the hybrid fields to be duplicated between the hybrid

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java

@@ -153,15 +153,14 @@ private static Object doClone(Object original) throws CloneNotSupportedException
         }
 
         // copy remaining non-object data
-        if (!isArrayLike) {
-            int objectSize = UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding));
-            int primitiveDataSize = objectSize - curOffset;
-            assert primitiveDataSize >= 0;
-            assert curOffset >= 0;
-            JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
-            curOffset += primitiveDataSize;
-            assert curOffset == objectSize;
-        }
+        int endOffset = isArrayLike ? LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding)
+                        : UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding));
+        int primitiveDataSize = endOffset - curOffset;
+        assert primitiveDataSize >= 0;
+        assert curOffset >= 0;
+        JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
+        curOffset += primitiveDataSize;
+        assert curOffset == endOffset;
 
         // reset monitor to uninitialized values
         int monitorOffset = hub.getMonitorOffset();
@@ -178,15 +177,15 @@ static boolean canVirtualize(ObjectClone node, VirtualizerTool tool) {
             return true;
         }
         ResolvedJavaType type = node.getConcreteType(alias.stamp(NodeView.DEFAULT));
-        if (type == null) {
-            return false;
-        }
-        if (!type.isArray() && type instanceof SharedType) {
+        if (type instanceof SharedType) {
             // Hybrids are instances with array-like encoding; cloning virtually is unimplemented.
             int encoding = ((SharedType) type).getHub().getLayoutEncoding();
-            return !LayoutEncoding.isArrayLike(encoding);
+            return !LayoutEncoding.isHybrid(encoding);
+        }
+        if (type != null && type.isArray()) {
+            return true; // cannot be a hybrid
         }
-        return true;
+        return false;
     }
 
     @NodeIntrinsic(value = ForeignCallNode.class)

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java

@@ -177,6 +177,19 @@ public Pod<T> build() {
             guaranteeUnbuilt();
             built = true;
 
+            /*
+             * We layout the requested fields in the hybrid object's array part in a similar fashion
+             * as UniverseBuilder does for regular instance fields, putting reference fields at the
+             * beginning and trying to put narrow fields in alignment gaps between wider fields. The
+             * entire layout of a pod might look as follows:
+             *
+             * [ hub pointer | identity hashcode | array length | superclass instance fields |
+             * instance fields | monitor | pod fields (ref, short, byte, long) | pod reference map ]
+             *
+             * The array length part would provide the combined length of the pod fields and pod
+             * reference map excluding any alignment padding at their beginning or the object's end.
+             */
+
             Collections.sort(fields);
 
             UnsignedWord baseOffset = LayoutEncoding.getArrayBaseOffset(
@@ -346,6 +359,20 @@ void add(int offset, int size) {
             bitset.set(index);
         }
 
+        /**
+         * Generates a reference map composed of unsigned byte pairs of {@code nrefs} (sequence of n
+         * references) and {@code gaps} (number of sequential reference-sized primitives). When a
+         * single lengthy sequence of references or primitives cannot be encoded in an unsigned
+         * byte, it is encoded in multiple pairs with either gaps or nrefs being 0. The reference
+         * map covers only the part of the array from the beginning until the last reference. The
+         * end of the reference map is indicated by a pair with {@code gaps} == 0, unless
+         * {@code nrefs} is 0xff which could also mean that the pair is part of an ongoing sequence,
+         * in which case an extra zero pair is added at the end.
+         *
+         * We try to place reference fields at the beginning, so the reference map begins with
+         * {@code nrefs} indicating the number of references right at the start of the array. If
+         * there are primitive fields at the beginning, this first value will be zero.
+         */
         byte[] encode() {
             if (bitset.isEmpty()) {
                 return new byte[]{0, 0};
@@ -354,7 +381,7 @@ byte[] encode() {
             ByteArrayOutputStream buffer = new ByteArrayOutputStream();
             int previous;
             int index = 0;
-            for (;;) { // assume references are placed first
+            for (;;) {
                 previous = index;
                 index = bitset.nextClearBit(previous);
                 int nrefs = index - previous;