[GR-48949] For instance objects, move the identity hashcode from the object header into the object.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotAllocationSnippets.java

@@ -24,9 +24,6 @@
  */
 package jdk.graal.compiler.hotspot.replacements;
 
-import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
-import static jdk.vm.ci.meta.DeoptimizationAction.None;
-import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
 import static jdk.graal.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
 import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT;
 import static jdk.graal.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_OPTIONVALUES;
@@ -73,6 +70,12 @@
 import static jdk.graal.compiler.replacements.ReplacementsUtil.staticAssert;
 import static jdk.graal.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
 import static jdk.graal.compiler.replacements.nodes.CStringConstant.cstring;
+import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
+import static jdk.vm.ci.meta.DeoptimizationAction.None;
+import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
+
+import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
 
 import jdk.graal.compiler.api.replacements.Fold;
 import jdk.graal.compiler.api.replacements.Fold.InjectedParameter;
@@ -118,9 +121,6 @@
 import jdk.graal.compiler.replacements.SnippetTemplate.Arguments;
 import jdk.graal.compiler.replacements.SnippetTemplate.SnippetInfo;
 import jdk.graal.compiler.word.Word;
-import org.graalvm.word.UnsignedWord;
-import org.graalvm.word.WordFactory;
-
 import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/AllocationSnippets.java

@@ -31,16 +31,17 @@
 import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability;
 import static jdk.graal.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
 
+import org.graalvm.word.LocationIdentity;
+import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
+
 import jdk.graal.compiler.nodes.PrefetchAllocateNode;
 import jdk.graal.compiler.nodes.extended.MembarNode;
 import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode;
 import jdk.graal.compiler.replacements.SnippetCounter.Group;
 import jdk.graal.compiler.replacements.nodes.ExplodeLoopNode;
 import jdk.graal.compiler.replacements.nodes.ZeroMemoryNode;
 import jdk.graal.compiler.word.Word;
-import org.graalvm.word.LocationIdentity;
-import org.graalvm.word.UnsignedWord;
-import org.graalvm.word.WordFactory;
 
 /**
  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
@@ -119,11 +120,11 @@ protected UnsignedWord arrayAllocationSize(int length, int arrayBaseOffset, int
         return WordFactory.unsigned(arrayAllocationSize(length, arrayBaseOffset, log2ElementSize, alignment));
     }
 
-    /**
-     * We do an unsigned multiplication so that a negative array length will result in an array size
-     * greater than Integer.MAX_VALUE.
-     */
-    public static long arrayAllocationSize(long length, int arrayBaseOffset, int log2ElementSize, int alignment) {
+    public static long arrayAllocationSize(int length, int arrayBaseOffset, int log2ElementSize, int alignment) {
+        /*
+         * We do an unsigned multiplication so that a negative array length will result in an array
+         * size greater than Integer.MAX_VALUE.
+         */
         long size = ((length & 0xFFFFFFFFL) << log2ElementSize) + arrayBaseOffset + (alignment - 1);
         long mask = ~(alignment - 1);
         return size & mask;

docs/reference-manual/native-image/DebugInfo.md

@@ -154,10 +154,10 @@ So, for example in the DWARF debug info model `java.lang.String` identifies a C+
 This class layout type declares the expected fields like `hash` of type `int` and `value` of type `byte[]` and methods like `String(byte[])`, `charAt(int)`, etc. However, the copy constructor which appears in Java as `String(String)` appears in `gdb` with the signature `String(java.lang.String *)`.
 
 The C++ layout class inherits fields and methods from class (layout) type `java.lang.Object` using C++ public inheritance.
-The latter in turn inherits standard oop (ordinary object pointer) header fields from a special struct class named `_objhdr` which includes two fields. The first field is called
-`hub` and its type is `java.lang.Class *` i.e. it is a pointer to the object's
-class. The second field is called `idHash` and has type `int`. It stores an
-identity hashcode for the object.
+The latter in turn inherits standard oop (ordinary object pointer) header fields from a special struct class named `_objhdr` which includes up to two fields (depending on the VM configuration).
+The first field is called `hub` and its type is `java.lang.Class *` i.e. it is a pointer to the object's class.
+The second field (optional) is called `idHash` and has type `int`.
+It stores an identity hashcode for the object.
 
 The `ptype` command can be used to print details of a specific type.
 Note that the Java type name must be specified in quotes because to escape the embedded `.` characters.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java

@@ -24,11 +24,6 @@
  */
 package com.oracle.svm.core.genscavenge;
 
-import jdk.graal.compiler.api.directives.GraalDirectives;
-import jdk.graal.compiler.api.replacements.Fold;
-import jdk.graal.compiler.replacements.ReplacementsUtil;
-import jdk.graal.compiler.word.ObjectAccess;
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.LocationIdentity;
@@ -51,6 +46,11 @@
 import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.api.directives.GraalDirectives;
+import jdk.graal.compiler.api.replacements.Fold;
+import jdk.graal.compiler.replacements.ReplacementsUtil;
+import jdk.graal.compiler.word.ObjectAccess;
+import jdk.graal.compiler.word.Word;
 import jdk.vm.ci.code.CodeUtil;
 
 /**
@@ -90,13 +90,13 @@ public final class ObjectHeaderImpl extends ObjectHeader {
         numAlignmentBits = CodeUtil.log2(ConfigurationValues.getObjectLayout().getAlignment());
         int numMinimumReservedBits = 3;
         VMError.guarantee(numMinimumReservedBits <= numAlignmentBits, "Minimum set of reserved bits must be provided by object alignment");
-        if (hasFixedIdentityHashField()) {
-            numReservedBits = numMinimumReservedBits;
-        } else {
+        if (isIdentityHashFieldOptional()) {
             VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable");
             numReservedBits = numMinimumReservedBits + 2;
             VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(),
                             "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits");
+        } else {
+            numReservedBits = numMinimumReservedBits;
         }
         numReservedExtraBits = numReservedBits - numAlignmentBits;
         reservedBitsMask = (1 << numReservedBits) - 1;
@@ -136,20 +136,22 @@ public Word encodeAsUnmanagedObjectHeader(DynamicHub hub) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     @Override
-    public void initializeHeaderOfNewObject(Pointer objectPointer, Word encodedHub) {
+    public void initializeHeaderOfNewObject(Pointer objectPointer, Word encodedHub, boolean isArrayLike) {
         ObjectLayout ol = ConfigurationValues.getObjectLayout();
+        boolean isIdentityHashFieldInObjectHeader = ol.isIdentityHashFieldInObjectHeader() || ol.isIdentityHashFieldAtTypeSpecificOffset() && isArrayLike;
         if (getReferenceSize() == Integer.BYTES) {
             dynamicAssert(encodedHub.and(WordFactory.unsigned(0xFFFFFFFF00000000L)).isNull(), "hub can only use 32 bits");
-            if (ol.hasFixedIdentityHashField()) {
-                dynamicAssert(ol.getFixedIdentityHashOffset() == getHubOffset() + 4, "assumed layout to optimize initializing write");
+            if (isIdentityHashFieldInObjectHeader) {
+                /* Use a single 64-bit write to initialize the hub and the identity hashcode. */
+                dynamicAssert(ol.getObjectHeaderIdentityHashOffset() == getHubOffset() + 4, "assumed layout to optimize initializing write");
                 objectPointer.writeLong(getHubOffset(), encodedHub.rawValue(), LocationIdentity.INIT_LOCATION);
             } else {
                 objectPointer.writeInt(getHubOffset(), (int) encodedHub.rawValue(), LocationIdentity.INIT_LOCATION);
             }
         } else {
             objectPointer.writeWord(getHubOffset(), encodedHub, LocationIdentity.INIT_LOCATION);
-            if (ol.hasFixedIdentityHashField()) {
-                objectPointer.writeInt(ol.getFixedIdentityHashOffset(), 0, LocationIdentity.INIT_LOCATION);
+            if (isIdentityHashFieldInObjectHeader) {
+                objectPointer.writeInt(ol.getObjectHeaderIdentityHashOffset(), 0, LocationIdentity.INIT_LOCATION);
             }
         }
     }
@@ -158,18 +160,19 @@ public void initializeHeaderOfNewObject(Pointer objectPointer, Word encodedHub)
     @Override
     public boolean hasOptionalIdentityHashField(Word header) {
         if (GraalDirectives.inIntrinsic()) {
-            ReplacementsUtil.staticAssert(!hasFixedIdentityHashField(), "use only when fields are not fixed");
+            ReplacementsUtil.staticAssert(isIdentityHashFieldOptional(), "use only when hashcode fields are optional");
         } else {
-            VMError.guarantee(!hasFixedIdentityHashField(), "use only when fields are not fixed");
+            VMError.guarantee(isIdentityHashFieldOptional(), "use only when hashcode fields are optional");
         }
+
         UnsignedWord inFieldState = IDHASH_STATE_IN_FIELD.shiftLeft(IDHASH_STATE_SHIFT);
         return header.and(IDHASH_STATE_BITS).equal(inFieldState);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void setIdentityHashInField(Object o) {
         assert VMOperation.isGCInProgress();
-        VMError.guarantee(!hasFixedIdentityHashField());
+        VMError.guarantee(isIdentityHashFieldOptional());
         UnsignedWord oldHeader = readHeaderFromObject(o);
         UnsignedWord inFieldState = IDHASH_STATE_IN_FIELD.shiftLeft(IDHASH_STATE_SHIFT);
         UnsignedWord newHeader = oldHeader.and(IDHASH_STATE_BITS.not()).or(inFieldState);
@@ -195,11 +198,12 @@ void setIdentityHashInField(Object o) {
     @Override
     public void setIdentityHashFromAddress(Pointer ptr, Word currentHeader) {
         if (GraalDirectives.inIntrinsic()) {
-            ReplacementsUtil.staticAssert(!hasFixedIdentityHashField(), "must always access field");
+            ReplacementsUtil.staticAssert(isIdentityHashFieldOptional(), "use only when hashcode fields are optional");
         } else {
-            VMError.guarantee(!hasFixedIdentityHashField());
-            assert !hasIdentityHashFromAddress(currentHeader);
+            assert isIdentityHashFieldOptional() : "use only when hashcode fields are optional";
+            assert !hasIdentityHashFromAddress(currentHeader) : "must not already have a hashcode";
         }
+
         UnsignedWord fromAddressState = IDHASH_STATE_FROM_ADDRESS.shiftLeft(IDHASH_STATE_SHIFT);
         UnsignedWord newHeader = currentHeader.and(IDHASH_STATE_BITS.not()).or(fromAddressState);
         writeHeaderToObject(ptr.toObjectNonNull(), newHeader);
@@ -217,13 +221,17 @@ public boolean hasIdentityHashFromAddress(Word header) {
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static boolean hasIdentityHashFromAddressInline(Word header) {
-        if (hasFixedIdentityHashField()) {
-            return false;
+        if (GraalDirectives.inIntrinsic()) {
+            ReplacementsUtil.staticAssert(isIdentityHashFieldOptional(), "use only when hashcode fields are optional");
+        } else {
+            assert isIdentityHashFieldOptional();
         }
+
         UnsignedWord fromAddressState = IDHASH_STATE_FROM_ADDRESS.shiftLeft(IDHASH_STATE_SHIFT);
         return header.and(IDHASH_STATE_BITS).equal(fromAddressState);
     }
 
+    @AlwaysInline(value = "Helper method that needs to be optimized away.")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static void dynamicAssert(boolean condition, String msg) {
         if (GraalDirectives.inIntrinsic()) {
@@ -309,7 +317,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro
                 assert obj.getPartition() instanceof FillerObjectDummyPartition;
             }
         }
-        if (!hasFixedIdentityHashField()) {
+        if (isIdentityHashFieldOptional()) {
             header |= (IDHASH_STATE_IN_FIELD.rawValue() << IDHASH_STATE_SHIFT);
         }
         return header;
@@ -427,7 +435,7 @@ static boolean hasShift() {
     }
 
     @Fold
-    static boolean hasFixedIdentityHashField() {
-        return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField();
+    static boolean isIdentityHashFieldOptional() {
+        return ConfigurationValues.getObjectLayout().isIdentityHashFieldOptional();
     }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -399,7 +399,7 @@ private Object copyAlignedObject(Object originalObj) {
         UnsignedWord originalSize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, false);
         UnsignedWord copySize = originalSize;
         boolean addIdentityHashField = false;
-        if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) {
+        if (ConfigurationValues.getObjectLayout().isIdentityHashFieldOptional()) {
             Word header = ObjectHeader.readHeaderFromObject(originalObj);
             if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) {
                 addIdentityHashField = true;
@@ -424,7 +424,7 @@ private Object copyAlignedObject(Object originalObj) {
         if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) {
             // Must do first: ensures correct object size below and in other places
             int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj);
-            int offset = LayoutEncoding.getOptionalIdentityHashOffset(copy);
+            int offset = LayoutEncoding.getIdentityHashOffset(copy);
             ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION);
             ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy);
         }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateTargetDescription.java

@@ -24,17 +24,16 @@
  */
 package com.oracle.svm.core;
 
+import java.util.EnumSet;
+
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
 import com.oracle.svm.core.code.RuntimeCodeCache;
-import com.oracle.svm.core.deopt.DeoptimizedFrame;
 
 import jdk.vm.ci.code.Architecture;
 import jdk.vm.ci.code.TargetDescription;
 
-import java.util.EnumSet;
-
 public class SubstrateTargetDescription extends TargetDescription {
     @Platforms(Platform.HOSTED_ONLY.class)
     public static boolean shouldInlineObjectsInImageCode() {
@@ -45,24 +44,14 @@ public static boolean shouldInlineObjectsInRuntimeCode() {
         return SubstrateOptions.SpawnIsolates.getValue() && RuntimeCodeCache.Options.WriteableCodeCache.getValue();
     }
 
-    private final int deoptScratchSpace;
     private final EnumSet<?> runtimeCheckedCPUFeatures;
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public SubstrateTargetDescription(Architecture arch, boolean isMP, int stackAlignment, int implicitNullCheckLimit, int deoptScratchSpace, EnumSet<?> runtimeCheckedCPUFeatures) {
+    public SubstrateTargetDescription(Architecture arch, boolean isMP, int stackAlignment, int implicitNullCheckLimit, EnumSet<?> runtimeCheckedCPUFeatures) {
         super(arch, isMP, stackAlignment, implicitNullCheckLimit, shouldInlineObjectsInImageCode());
-        this.deoptScratchSpace = deoptScratchSpace;
         this.runtimeCheckedCPUFeatures = runtimeCheckedCPUFeatures;
     }
 
-    /**
-     * Returns the amount of scratch space which must be reserved for return value registers in
-     * {@link DeoptimizedFrame}.
-     */
-    public int getDeoptScratchSpace() {
-        return deoptScratchSpace;
-    }
-
     public EnumSet<?> getRuntimeCheckedCPUFeatures() {
         return runtimeCheckedCPUFeatures;
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java

@@ -28,8 +28,6 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
-import jdk.graal.compiler.nodes.java.ArrayLengthNode;
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -56,6 +54,9 @@
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.nodes.java.ArrayLengthNode;
+import jdk.graal.compiler.word.Word;
+
 /**
  * Support for allocating and accessing non-moving arrays. Such arrays are safe to access during
  * garbage collection. They can also be passed between isolates, provided that they do not contain
@@ -99,7 +100,7 @@ private static <T extends NonmovableArray<?>> T createArray(int length, Class<?>
 
         ObjectHeader header = Heap.getHeap().getObjectHeader();
         Word encodedHeader = header.encodeAsUnmanagedObjectHeader(hub);
-        header.initializeHeaderOfNewObject(array, encodedHeader);
+        header.initializeHeaderOfNewObject(array, encodedHeader, true);
 
         array.writeInt(ConfigurationValues.getObjectLayout().getArrayLengthOffset(), length);
         // already zero-initialized thanks to calloc()