diff --git a/substratevm/mx.substratevm/testhello.py b/substratevm/mx.substratevm/testhello.py index 8e0e63002740..59014435735f 100644 --- a/substratevm/mx.substratevm/testhello.py +++ b/substratevm/mx.substratevm/testhello.py @@ -57,7 +57,7 @@ def __init__(self, name, regexps): self.name = name if not isinstance(regexps, list): regexps = [regexps] - self.rexps = [re.compile(regexp) for regexp in regexps] + self.rexps = [re.compile(r) for r in regexps if r is not None] # Check that successive lines of a gdb command's output text # match the corresponding regexp patterns provided when this @@ -180,6 +180,9 @@ def test(): # disable printing of address symbols execute("set print symbol off") + hub_ref_size = int(execute("printf \"%d\", sizeof('java.lang.Object'::__hub__)")) + fixed_idhash_field = (hub_ref_size > 4) + # Print DefaultGreeter and check the modifiers of its methods and fields exec_string = execute("ptype 'hello.Hello$DefaultGreeter'") rexp = [r"type = class hello\.Hello\$DefaultGreeter : public hello\.Hello\$Greeter {", @@ -241,7 +244,7 @@ def test(): r"%s = {"%(spaces_pattern), r"%s<_objhdr> = {"%(spaces_pattern), r"%shub = %s,"%(spaces_pattern, address_pattern), - r"%sidHash = %s"%(spaces_pattern, address_pattern), + r"%sidHash = %s"%(spaces_pattern, address_pattern) if fixed_idhash_field else None, r"%s}, }, "%(spaces_pattern), r"%smembers of java\.lang\.String\[\]:"%(spaces_pattern), r"%slen = 0x0,"%(spaces_pattern), @@ -260,7 +263,7 @@ def test(): r"%s = {"%(spaces_pattern), r"%s<_objhdr> = {"%(spaces_pattern), r"%shub = %s,"%(spaces_pattern, address_pattern), - r"%sidHash = %s"%(spaces_pattern, address_pattern), + r"%sidHash = %s"%(spaces_pattern, address_pattern) if fixed_idhash_field else None, r"%s}, },"%(spaces_pattern), r"%smembers of java\.lang\.Class:"%(spaces_pattern), r"%sname = %s,"%(spaces_pattern, address_pattern), @@ -270,7 +273,7 @@ def test(): r"%s = {"%(spaces_pattern), r"%s<_objhdr> = {"%(spaces_pattern), r"%shub = %s,"%(spaces_pattern, address_pattern), - r"%sidHash = %s"%(spaces_pattern, address_pattern), + r"%sidHash = %s"%(spaces_pattern, address_pattern) if fixed_idhash_field else None, r"%s}, },"%(spaces_pattern), r"%smembers of java\.lang\.Class:"%(spaces_pattern), r"%sname = %s,"%(spaces_pattern, address_pattern), @@ -310,7 +313,7 @@ def test(): r"%s = {"%(spaces_pattern), r"%s<_objhdr> = {"%(spaces_pattern), r"%shub = %s,"%(spaces_pattern, address_pattern), - r"%sidHash = %s"%(spaces_pattern, address_pattern), + r"%sidHash = %s"%(spaces_pattern, address_pattern) if fixed_idhash_field else None, r"%s}, },"%(spaces_pattern), r"%smembers of java\.lang\.Class:"%(spaces_pattern), r"%sname = %s,"%(spaces_pattern, address_pattern), @@ -426,12 +429,12 @@ def test(): if isolates: rexp = [r"type = struct _objhdr {", r"%s_z_\.java\.lang\.Class \*hub;"%(spaces_pattern), - r"%sint idHash;"%(spaces_pattern), + r"%sint idHash;"%(spaces_pattern) if fixed_idhash_field else None, r"}"] else: rexp = [r"type = struct _objhdr {", r"%sjava\.lang\.Class \*hub;"%(spaces_pattern), - r"%sint idHash;"%(spaces_pattern), + r"%sint idHash;"%(spaces_pattern) if fixed_idhash_field else None, r"}"] checker = Checker('ptype _objhdr', rexp) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index fc7055b66a41..fccc52eb5d1f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -263,6 +263,17 @@ public int getTenuringAge() { return tenuringThreshold; } + @Override + public void onCollectionBegin(boolean completeCollection, long requestingNanoTime) { + // Capture the fraction of bytes in aligned chunks at the start to include all allocated + // (also dead) objects, because we use it to reserve aligned chunks for future allocations + UnsignedWord youngChunkBytes = GCImpl.getGCImpl().getAccounting().getYoungChunkBytesBefore(); + if (youngChunkBytes.notEqual(0)) { + UnsignedWord youngAlignedChunkBytes = HeapImpl.getHeapImpl().getYoungGeneration().getAlignedChunkBytes(); + avgYoungGenAlignedChunkFraction.sample(UnsignedUtils.toDouble(youngAlignedChunkBytes) / UnsignedUtils.toDouble(youngChunkBytes)); + } + } + @Override public UnsignedWord getMinimumHeapSize() { return sizes.minHeapSize; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java index bd16a39b9df6..fe3c3e78b945 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java @@ -94,11 +94,15 @@ class AdaptiveCollectionPolicy extends AbstractCollectionPolicy { * (disabled by default) which uses linear least-square fitting without discounting. */ private static final boolean ADAPTIVE_SIZE_USE_COST_ESTIMATORS = true; + /** Number of space size adjustments before cost estimators are used. HotSpot default: 20. */ private static final int ADAPTIVE_SIZE_POLICY_INITIALIZING_STEPS = ADAPTIVE_SIZE_POLICY_READY_THRESHOLD; - /** The minimum increase in throughput in percent for expanding a space by 1% of its size. */ - private static final double ADAPTIVE_SIZE_ESTIMATOR_MIN_SIZE_COST_TRADEOFF = 0.5; + /** + * The minimum estimated decrease in {@link #gcCost()} in percent to decide in favor of + * expanding a space by 1% of the combined size of {@link #edenSize} and {@link #promoSize}. + */ + private static final double ADAPTIVE_SIZE_ESTIMATOR_MIN_TOTAL_SIZE_COST_TRADEOFF = 0.5; /** The effective number of most recent data points used by estimator (exponential decay). */ - private static final int ADAPTIVE_SIZE_COST_ESTIMATORS_HISTORY_LENGTH = 12; + private static final double ADAPTIVE_SIZE_COST_ESTIMATORS_HISTORY_LENGTH = 12; /** Threshold for triggering a complete collection after repeated minor collections. */ private static final int CONSECUTIVE_MINOR_TO_MAJOR_COLLECTION_PAUSE_TIME_RATIO = 2; /** @@ -240,7 +244,7 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean completeCollection, @SuppressWarnings("unused") GCCause cause) { boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(youngGenChangeForMinorThroughput, minorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesCost(minorCostEstimator, edenSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, edenSize, majorGcCost(), promoSize); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. @@ -282,15 +286,17 @@ private static boolean shouldUseEstimator(long genChangeForThroughput, double co return ADAPTIVE_SIZE_USE_COST_ESTIMATORS && genChangeForThroughput > ADAPTIVE_SIZE_POLICY_INITIALIZING_STEPS && cost <= ADAPTIVE_SIZE_COST_ESTIMATOR_GC_COST_LIMIT; } - private static boolean expansionSignificantlyReducesCost(ReciprocalLeastSquareFit estimator, UnsignedWord size) { + private static boolean expansionSignificantlyReducesTotalCost(ReciprocalLeastSquareFit estimator, UnsignedWord size, double otherCost, UnsignedWord otherSize) { + double totalSize = UnsignedUtils.toDouble(size.add(otherSize)); double x0 = UnsignedUtils.toDouble(size); double deltax = (1.01 - 1) * x0; - if (deltax == 0) { // division by zero below - return false; + if (deltax == 0 || totalSize == 0) { // division by zero below + return true; // general assumption for space expansion } - double y0 = estimator.estimate(x0); - double y1 = y0 * (1 - 0.01 * ADAPTIVE_SIZE_ESTIMATOR_MIN_SIZE_COST_TRADEOFF); + double y0 = estimator.estimate(x0) + otherCost; + double y1 = y0 * (1 - deltax / totalSize * ADAPTIVE_SIZE_ESTIMATOR_MIN_TOTAL_SIZE_COST_TRADEOFF); double minSlope = (y1 - y0) / deltax; + double estimatedSlope = estimator.getSlope(x0); return estimatedSlope <= minSlope; } @@ -383,16 +389,10 @@ public void onCollectionBegin(boolean completeCollection, long requestingNanoTim latestMinorMutatorIntervalNanos = timer.getMeasuredNanos(); } - // Capture the fraction of bytes in aligned chunks at the start to include all allocated - // (also dead) objects, because we use it to reserve aligned chunks for future allocations - UnsignedWord youngChunkBytes = GCImpl.getGCImpl().getAccounting().getYoungChunkBytesBefore(); - if (youngChunkBytes.notEqual(0)) { - UnsignedWord youngAlignedChunkBytes = HeapImpl.getHeapImpl().getYoungGeneration().getAlignedChunkBytes(); - avgYoungGenAlignedChunkFraction.sample(UnsignedUtils.toDouble(youngAlignedChunkBytes) / UnsignedUtils.toDouble(youngChunkBytes)); - } - timer.reset(); timer.open(); // measure collection pause + + super.onCollectionBegin(completeCollection, requestingNanoTime); } @Override @@ -458,7 +458,7 @@ private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_f boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(oldGenChangeForMajorThroughput, majorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesCost(majorCostEstimator, promoSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, promoSize, minorGcCost(), edenSize); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java index b4796cc84061..5e60f8f6c52b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java @@ -39,17 +39,34 @@ class AdaptiveWeightedAverage { static final int OLD_THRESHOLD = 100; - private final int weight; + /** @see #computeEffectiveHistoryLengthForWeight */ + static double computeWeightForEffectiveHistoryLength(double length) { + assert length > 0; + return 100.0 * (1.0 - Math.pow(Math.E, -1.0 / length)); + } + + /** + * Computes the effective history length for the given weight, which is the number of data + * points after which the former history is discounted to 1/e, i.e., its time constant. + */ + static double computeEffectiveHistoryLengthForWeight(double weight) { + assert weight > 0 && weight <= 100; + return -1.0 / Math.log(1.0 - weight / 100.0); + } + + private final double weight; private double average; private long sampleCount; private boolean isOld; - AdaptiveWeightedAverage(int weight) { + /** @param weight Weight of newest sample in percent, from 0 (exclusive) to 100 (inclusive). */ + AdaptiveWeightedAverage(double weight) { this(weight, 0); } - AdaptiveWeightedAverage(int weight, double avg) { + AdaptiveWeightedAverage(double weight, double avg) { + assert weight > 0 && weight <= 100; this.weight = weight; this.average = avg; } @@ -76,16 +93,16 @@ protected double computeAdaptiveAverage(double sample, double avg) { * it meaningful. We'd like the first weight used to be 1, the second to be 1/2, etc until * we have OLD_THRESHOLD/weight samples. */ - long countWeight = 0; + double countWeight = 0; if (!isOld) { // avoid division by zero if the counter wraps - countWeight = OLD_THRESHOLD / sampleCount; + countWeight = OLD_THRESHOLD / (double) sampleCount; } - long adaptiveWeight = Math.max(weight, countWeight); + double adaptiveWeight = Math.max(weight, countWeight); return expAvg(avg, sample, adaptiveWeight); } - private static double expAvg(double avg, double sample, long adaptiveWeight) { - assert adaptiveWeight <= 100 : "weight must be a percentage"; + private static double expAvg(double avg, double sample, double adaptiveWeight) { + assert adaptiveWeight > 0 && adaptiveWeight <= 100 : "weight must be a percentage"; return (100.0 - adaptiveWeight) * avg / 100.0 + adaptiveWeight * sample / 100.0; } } @@ -103,7 +120,7 @@ class AdaptivePaddedAverage extends AdaptiveWeightedAverage { private double paddedAverage; private double deviation; - AdaptivePaddedAverage(int weight, int padding) { + AdaptivePaddedAverage(double weight, int padding) { this(weight, padding, false); } @@ -112,7 +129,7 @@ class AdaptivePaddedAverage extends AdaptiveWeightedAverage { * allowed to change. This is to prevent zero samples from drastically changing the * padded average. */ - AdaptivePaddedAverage(int weight, int padding, boolean noZeroDeviations) { + AdaptivePaddedAverage(double weight, int padding, boolean noZeroDeviations) { super(weight); this.padding = padding; this.noZeroDeviations = noZeroDeviations; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index f689a2b96404..bc15be90bdcf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -117,11 +117,13 @@ static UnsignedWord getCommittedObjectMemory(AlignedHeader that) { return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static AlignedHeader getEnclosingChunk(Object obj) { Pointer ptr = Word.objectToUntrackedPointer(obj); return getEnclosingChunkFromObjectPointer(ptr); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { return (AlignedHeader) PointerUtils.roundDown(ptr, HeapParameters.getAlignedHeapChunkAlignment()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java index 4902e85fd9cd..5a2f51ad37fc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapAllocator.java @@ -179,7 +179,7 @@ public long getUnallocatedBytes() { this.position = position; /* Cache to prevent frequent lookups of the object layout from ImageSingletons. */ - minimumObjectSize = ConfigurationValues.getObjectLayout().getMinimumObjectSize(); + this.minimumObjectSize = ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize(); } public long getPosition() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java index 45e387f62a18..1e23506521d0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java @@ -57,7 +57,7 @@ public class ChunkedImageHeapPartition extends AbstractImageHeapPartition { this.hugeObjects = hugeObjects; /* Cache to prevent frequent lookups of the object layout from ImageSingletons. */ - minimumObjectSize = ConfigurationValues.getObjectLayout().getMinimumObjectSize(); + this.minimumObjectSize = ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize(); } boolean usesUnalignedObjects() { @@ -134,7 +134,7 @@ private static NavigableMap> createSortedObjectsMap for (ImageHeapObject obj : sorted) { long objSize = obj.getSize(); if (objSize != currentObjectsSize) { - assert objSize > currentObjectsSize && objSize >= ConfigurationValues.getObjectLayout().getMinimumObjectSize(); + assert objSize > currentObjectsSize && objSize >= ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize(); currentObjectsSize = objSize; currentQueue = new ArrayDeque<>(); map.put(currentObjectsSize, currentQueue); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 7a726e002eaf..fff7b6482740 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -32,6 +32,7 @@ import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.util.ReflectionUtil; /** The interface for a garbage collection policy. All sizes are in bytes. */ public interface CollectionPolicy { @@ -53,21 +54,27 @@ static String getInitialPolicyName() { @Platforms(Platform.HOSTED_ONLY.class) static CollectionPolicy getInitialPolicy() { String name = getInitialPolicyName(); + Class clazz = getPolicyClass(name); + return ReflectionUtil.newInstance(clazz); + } + + @Platforms(Platform.HOSTED_ONLY.class) + static Class getPolicyClass(String name) { switch (name) { case "Adaptive": - return new AdaptiveCollectionPolicy(); + return AdaptiveCollectionPolicy.class; case "AggressiveShrink": - return new AggressiveShrinkCollectionPolicy(); + return AggressiveShrinkCollectionPolicy.class; case "Proportionate": - return new ProportionateSpacesPolicy(); + return ProportionateSpacesPolicy.class; case "BySpaceAndTime": - return new BasicCollectionPolicies.BySpaceAndTime(); + return BasicCollectionPolicies.BySpaceAndTime.class; case "OnlyCompletely": - return new BasicCollectionPolicies.OnlyCompletely(); + return BasicCollectionPolicies.OnlyCompletely.class; case "OnlyIncrementally": - return new BasicCollectionPolicies.OnlyIncrementally(); + return BasicCollectionPolicies.OnlyIncrementally.class; case "NeverCollect": - return new BasicCollectionPolicies.NeverCollect(); + return BasicCollectionPolicies.NeverCollect.class; } throw UserError.abort("Policy %s does not exist.", name); } @@ -75,10 +82,10 @@ static CollectionPolicy getInitialPolicy() { @Platforms(Platform.HOSTED_ONLY.class) static int getMaxSurvivorSpaces(Integer userValue) { String name = getInitialPolicyName(); - if ("Adaptive".equals(name) || "Proportionate".equals(name)) { - return AbstractCollectionPolicy.getMaxSurvivorSpaces(userValue); + if (BasicCollectionPolicies.BasicPolicy.class.isAssignableFrom(getPolicyClass(name))) { + return BasicCollectionPolicies.getMaxSurvivorSpaces(userValue); } - return BasicCollectionPolicies.getMaxSurvivorSpaces(userValue); + return AbstractCollectionPolicy.getMaxSurvivorSpaces(userValue); } static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 979d3de9b598..58bb66d00afc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -196,7 +196,6 @@ private void collectOperation(CollectionVMOperationData data) { startCollectionOrExit(); timers.resetAllExceptMutator(); - collectionEpoch = collectionEpoch.add(1); /* Flush all TLAB chunks to eden. */ ThreadLocalAllocation.disableAndFlushForAllThreads(); @@ -358,7 +357,7 @@ private void printGCBefore(String cause) { Log verboseGCLog = Log.log(); HeapImpl heap = HeapImpl.getHeapImpl(); sizeBefore = ((SubstrateGCOptions.PrintGC.getValue() || SerialGCOptions.PrintHeapShape.getValue()) ? getChunkBytes() : WordFactory.zero()); - if (SubstrateGCOptions.VerboseGC.getValue() && getCollectionEpoch().equal(1)) { + if (SubstrateGCOptions.VerboseGC.getValue() && getCollectionEpoch().equal(0)) { verboseGCLog.string("[Heap policy parameters: ").newline(); verboseGCLog.string(" YoungGenerationSize: ").unsigned(getPolicy().getMaximumYoungGenerationSize()).newline(); verboseGCLog.string(" MaximumHeapSize: ").unsigned(getPolicy().getMaximumHeapSize()).newline(); @@ -1151,6 +1150,7 @@ private void startCollectionOrExit() { private void finishCollection() { assert collectionInProgress; + collectionEpoch = collectionEpoch.add(1); collectionInProgress = false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index b5b0d7f33574..03105a9b19fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -28,7 +28,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -86,13 +85,14 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // This is the most expensive check as it accesses the heap fairly randomly, which results // in a lot of cache misses. - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(p); + ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); + Word header = ohi.readHeaderFromPointer(p); if (GCImpl.getGCImpl().isCompleteCollection() || !RememberedSet.get().hasRememberedSet(header)) { if (ObjectHeaderImpl.isForwardedHeader(header)) { counters.noteForwardedReferent(); // Update the reference to point to the forwarded Object. - Object obj = ObjectHeaderImpl.getForwardedObject(p, header); + Object obj = ohi.getForwardedObject(p, header); Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed); RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); @@ -101,7 +101,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); - assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); + assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 9baa06919ec8..69104f59be47 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,6 +26,7 @@ import java.util.function.IntUnaryOperator; +import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -33,6 +34,7 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity; import org.graalvm.word.ComparableWord; +import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.SignedWord; @@ -46,6 +48,7 @@ import com.oracle.svm.core.c.struct.PinnedObjectField; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.option.HostedOptionKey; /** @@ -166,6 +169,12 @@ public interface Header> extends HeaderPadding { @RawField @UniqueLocationIdentity void setOffsetToNextChunk(SignedWord newNext); + + @RawField + UnsignedWord getIdentityHashSalt(LocationIdentity identity); + + @RawField + void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { @@ -174,6 +183,13 @@ public static void initialize(Header chunk, Pointer objectsStart, UnsignedWor HeapChunk.setSpace(chunk, null); HeapChunk.setNext(chunk, WordFactory.nullPointer()); HeapChunk.setPrevious(chunk, WordFactory.nullPointer()); + + /* + * The epoch is obviously not random, but cheap to use, and we cannot use a random number + * generator object in all contexts where we are called from, particularly during GC. + * Together with a good bit mixer function, it seems sufficient. + */ + HeapChunk.setIdentityHashSalt(chunk, GCImpl.getGCImpl().getCollectionEpoch()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -246,6 +262,16 @@ public static > void setNext(Header that, T newNext) { that.setOffsetToNextChunk(offsetFromPointer(that, newNext)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getIdentityHashSalt(Header that) { + return that.getIdentityHashSalt(IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setIdentityHashSalt(Header that, UnsignedWord value) { + that.setIdentityHashSalt(value, IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); + } + /** * Converts from an offset to a pointer, where a zero offset translates to {@code NULL}. This is * necessary for treating image heap chunks, where addresses at runtime are not yet known. @@ -287,7 +313,7 @@ public static boolean walkObjectsFromInline(Header that, Pointer startOffset, if (!visitor.visitObjectInline(obj)) { return false; } - offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); + offset = offset.add(LayoutEncoding.getSizeFromObjectInlineInGC(obj)); } return true; } @@ -302,13 +328,18 @@ public static Pointer asPointer(Header that) { return (Pointer) that; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static HeapChunk.Header getEnclosingHeapChunk(Object obj) { - assert !HeapImpl.getHeapImpl().isInImageHeap(obj) || HeapImpl.usesImageHeapChunks() : "Must be checked before calling this method"; - assert !ObjectHeaderImpl.isPointerToForwardedObject(Word.objectToUntrackedPointer(obj)) : "Forwarded objects must be a pointer and not an object"; + if (!GraalDirectives.inIntrinsic()) { + assert !HeapImpl.getHeapImpl().isInImageHeap(obj) || HeapImpl.usesImageHeapChunks() : "Must be checked before calling this method"; + assert !ObjectHeaderImpl.getObjectHeaderImpl().isPointerToForwardedObject(Word.objectToUntrackedPointer(obj)) : "Forwarded objects must be a pointer and not an object"; + } if (ObjectHeaderImpl.isAlignedObject(obj)) { return AlignedHeapChunk.getEnclosingChunk(obj); } else { - assert ObjectHeaderImpl.isUnalignedObject(obj); + if (!GraalDirectives.inIntrinsic()) { + assert ObjectHeaderImpl.isUnalignedObject(obj); + } return UnalignedHeapChunk.getEnclosingChunk(obj); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 69f4b5373efb..0bd8ec0fae7c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.SuppressFBWarnings; @@ -686,6 +687,16 @@ public void dirtyAllReferencesOf(Object obj) { } } + @Override + @Uninterruptible(reason = "Ensure that no GC can move the object to another chunk.", callerMustBe = true) + public long getIdentityHashSalt(Object obj) { + if (!GraalDirectives.inIntrinsic()) { + assert !isInImageHeap(obj) : "Image heap objects have identity hash code fields"; + } + HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); + return HeapChunk.getIdentityHashSalt(chunk).rawValue(); + } + static Pointer getImageHeapStart() { int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace(); if (imageHeapOffsetInAddressSpace > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index 20e3678afd23..38af545a2aad 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -30,7 +30,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.MemoryWalker; @@ -258,7 +257,7 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH return false; } - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(ptr); + Word header = ObjectHeaderImpl.getObjectHeaderImpl().readHeaderFromPointer(ptr); if (ObjectHeaderImpl.isProducedHeapChunkZapped(header) || ObjectHeaderImpl.isConsumedHeapChunkZapped(header)) { Log.log().string("Object ").zhex(ptr).string(" has a zapped header: ").zhex(header).newline(); return false; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index dac4008c4d59..44d04a7cdd15 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -196,11 +196,11 @@ public UnalignedHeader getFirstWritableUnalignedChunk() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static Word getObjectEnd(Object obj) { + private static Pointer getObjectEnd(Object obj) { if (obj == null) { return WordFactory.nullPointer(); } - return Word.objectToUntrackedPointer(obj).add(LayoutEncoding.getSizeFromObject(obj)); + return LayoutEncoding.getImageHeapObjectEnd(obj); } @SuppressWarnings("unchecked") diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index d035d94df0fe..2c17d3bbf737 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -119,7 +119,7 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject } else if (!visitor.visitObject(currentObject)) { return false; } - current = LayoutEncoding.getObjectEnd(current.toObject()); + current = LayoutEncoding.getImageHeapObjectEnd(current.toObject()); } if (HeapImpl.usesImageHeapChunks() && current.belowThan(lastPointer)) { currentChunk = HeapChunk.getNext(currentChunk); @@ -157,7 +157,7 @@ public UnsignedWord getSize(ImageHeapInfo info) { if (firstStart.isNull()) { // no objects return WordFactory.zero(); } - Pointer lastEnd = LayoutEncoding.getObjectEnd(getLastObject(info)); + Pointer lastEnd = LayoutEncoding.getImageHeapObjectEnd(getLastObject(info)); return lastEnd.subtract(firstStart); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 1a930ece064d..fa9e1d953fe7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -26,11 +26,9 @@ import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.compiler.core.common.CompressEncoding; import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.LocationIdentity; @@ -50,8 +48,11 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.image.ImageHeapObject; import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.VMError; +import jdk.vm.ci.code.CodeUtil; + /** * The pointer to the hub is either an uncompressed absolute reference or a heap-base-relative * reference without a shift. This limits the address space where all hubs must be placed to 32/64 @@ -62,16 +63,43 @@ * {@link Heap#isInImageHeap}. */ public final class ObjectHeaderImpl extends ObjectHeader { - private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b001); - private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b010); - private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b100); + private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b00001); + private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b00010); + private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b00100); + + /** + * Optional: per-object identity hash code state to avoid a fixed field, initially implicitly + * initialized to {@link #IDHASH_STATE_UNASSIGNED}. + */ + private static final int IDHASH_STATE_SHIFT = 3; + private static final UnsignedWord IDHASH_STATE_BITS = WordFactory.unsigned(0b11000); - private static final int RESERVED_BITS_MASK = 0b111; - private static final UnsignedWord MASK_HEADER_BITS = WordFactory.unsigned(RESERVED_BITS_MASK); - private static final UnsignedWord CLEAR_HEADER_BITS = MASK_HEADER_BITS.not(); + @SuppressWarnings("unused") // + private static final UnsignedWord IDHASH_STATE_UNASSIGNED = WordFactory.unsigned(0b00); + private static final UnsignedWord IDHASH_STATE_FROM_ADDRESS = WordFactory.unsigned(0b01); + private static final UnsignedWord IDHASH_STATE_IN_FIELD = WordFactory.unsigned(0b10); + + private final int numReservedBits; + private final int numAlignmentBits; + private final int numReservedExtraBits; + + private final int reservedBitsMask; @Platforms(Platform.HOSTED_ONLY.class) ObjectHeaderImpl() { + numAlignmentBits = CodeUtil.log2(ConfigurationValues.getObjectLayout().getAlignment()); + int numMinimumReservedBits = 3; + VMError.guarantee(numMinimumReservedBits <= numAlignmentBits, "Minimum set of reserved bits must be provided by object alignment"); + if (hasFixedIdentityHashField()) { + numReservedBits = numMinimumReservedBits; + } else { + VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); + numReservedBits = numMinimumReservedBits + 2; + VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); + } + numReservedExtraBits = numReservedBits - numAlignmentBits; + reservedBitsMask = (1 << numReservedBits) - 1; } @Fold @@ -83,9 +111,7 @@ public static ObjectHeaderImpl getObjectHeaderImpl() { @Override public int getReservedBitsMask() { - assert MASK_HEADER_BITS.rawValue() == RESERVED_BITS_MASK; - assert CLEAR_HEADER_BITS.rawValue() == ~RESERVED_BITS_MASK; - return RESERVED_BITS_MASK; + return reservedBitsMask; } /** @@ -93,8 +119,9 @@ public int getReservedBitsMask() { * enabled, the specified address must be the uncompressed absolute address of the object in * memory. */ + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord readHeaderFromPointer(Pointer objectPointer) { + public Word readHeaderFromPointer(Pointer objectPointer) { if (getReferenceSize() == Integer.BYTES) { return WordFactory.unsigned(objectPointer.readInt(getHubOffset())); } else { @@ -103,7 +130,7 @@ public static UnsignedWord readHeaderFromPointer(Pointer objectPointer) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord readHeaderFromObject(Object o) { + public static Word readHeaderFromObject(Object o) { if (getReferenceSize() == Integer.BYTES) { return WordFactory.unsigned(ObjectAccess.readInt(o, getHubOffset())); } else { @@ -114,34 +141,31 @@ public static UnsignedWord readHeaderFromObject(Object o) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public DynamicHub readDynamicHubFromPointer(Pointer ptr) { - UnsignedWord header = readHeaderFromPointer(ptr); + Word header = readHeaderFromPointer(ptr); return dynamicHubFromObjectHeader(header); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public DynamicHub dynamicHubFromObjectHeader(UnsignedWord header) { - UnsignedWord pointerBits = clearBits(header); - Object objectValue; - ReferenceAccess referenceAccess = ReferenceAccess.singleton(); - if (referenceAccess.haveCompressedReferences()) { - UnsignedWord compressedBits = pointerBits.unsignedShiftRight(getCompressionShift()); - objectValue = referenceAccess.uncompressReference(compressedBits); - } else { - objectValue = ((Pointer) pointerBits).toObject(); - } - return (DynamicHub) objectValue; + public DynamicHub dynamicHubFromObjectHeader(Word header) { + return (DynamicHub) extractPotentialDynamicHubFromHeader(header).toObject(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Pointer readPotentialDynamicHubFromPointer(Pointer ptr) { - UnsignedWord potentialHeader = ObjectHeaderImpl.readHeaderFromPointer(ptr); - UnsignedWord pointerBits = clearBits(potentialHeader); + Word potentialHeader = readHeaderFromPointer(ptr); + return extractPotentialDynamicHubFromHeader(potentialHeader); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer extractPotentialDynamicHubFromHeader(UnsignedWord header) { if (ReferenceAccess.singleton().haveCompressedReferences()) { - UnsignedWord compressedBits = pointerBits.unsignedShiftRight(ObjectHeader.getCompressionShift()); - return KnownIntrinsics.heapBase().add(compressedBits.shiftLeft(ObjectHeader.getCompressionShift())); + UnsignedWord hubBits = header.unsignedShiftRight(numReservedBits); + UnsignedWord baseRelativeBits = hubBits.shiftLeft(numAlignmentBits); + return KnownIntrinsics.heapBase().add(baseRelativeBits); } else { + UnsignedWord pointerBits = clearBits(header); return (Pointer) pointerBits; } } @@ -156,15 +180,90 @@ public Word encodeAsUnmanagedObjectHeader(DynamicHub hub) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public void initializeHeaderOfNewObject(Pointer objectPointer, Word encodedHub) { + ObjectLayout ol = ConfigurationValues.getObjectLayout(); if (getReferenceSize() == Integer.BYTES) { - dynamicAssert(getIdentityHashCodeOffset() == getHubOffset() + 4, "assumed layout to optimize initializing write"); - dynamicAssert(encodedHub.and(WordFactory.unsigned(0xFFFFFFFF00000000L)).isNull(), "hub can only use 32 bit"); - - objectPointer.writeLong(getHubOffset(), encodedHub.rawValue(), LocationIdentity.INIT_LOCATION); + dynamicAssert(encodedHub.and(WordFactory.unsigned(0xFFFFFFFF00000000L)).isNull(), "hub can only use 32 bits"); + if (ol.hasFixedIdentityHashField()) { + dynamicAssert(ol.getFixedIdentityHashOffset() == getHubOffset() + 4, "assumed layout to optimize initializing write"); + objectPointer.writeLong(getHubOffset(), encodedHub.rawValue(), LocationIdentity.INIT_LOCATION); + } else { + objectPointer.writeInt(getHubOffset(), (int) encodedHub.rawValue(), LocationIdentity.INIT_LOCATION); + } } else { objectPointer.writeWord(getHubOffset(), encodedHub, LocationIdentity.INIT_LOCATION); - objectPointer.writeInt(getIdentityHashCodeOffset(), 0, LocationIdentity.INIT_LOCATION); + if (ol.hasFixedIdentityHashField()) { + objectPointer.writeInt(ol.getFixedIdentityHashOffset(), 0, LocationIdentity.INIT_LOCATION); + } + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean hasOptionalIdentityHashField(Word header) { + if (GraalDirectives.inIntrinsic()) { + ReplacementsUtil.staticAssert(!hasFixedIdentityHashField(), "use only when fields are not fixed"); + } else { + VMError.guarantee(!hasFixedIdentityHashField(), "use only when fields are not fixed"); + } + UnsignedWord inFieldState = IDHASH_STATE_IN_FIELD.shiftLeft(IDHASH_STATE_SHIFT); + return header.and(IDHASH_STATE_BITS).equal(inFieldState); + } + + void setIdentityHashInField(Object o) { + assert VMOperation.isGCInProgress(); + VMError.guarantee(!hasFixedIdentityHashField()); + UnsignedWord oldHeader = readHeaderFromObject(o); + UnsignedWord inFieldState = IDHASH_STATE_IN_FIELD.shiftLeft(IDHASH_STATE_SHIFT); + UnsignedWord newHeader = oldHeader.and(IDHASH_STATE_BITS.not()).or(inFieldState); + writeHeaderToObject(o, newHeader); + assert hasOptionalIdentityHashField(readHeaderFromObject(o)); + } + + /** + * Set bits in an object's header to indicate that it has been assigned an identity hash code + * that is based on its current address of the time of the call. + * + * This (currently) does not need to use atomic instructions because two threads can only modify + * the header in the same way independent of each other. If this changes in the future by the + * introduction of other header bits or by changes to the identity hash code states and their + * transitions, this needs to be reconsidered. + * + * Still, this method and the caller that computes the identity hash code need to be + * uninterruptible (atomic with regard to GC) so that no GC can occur and unexpectedly move the + * object (changing its potential identity hash code), modify its object header, or introduce an + * identity hash code field. + */ + @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.", callerMustBe = true) + @Override + public void setIdentityHashFromAddress(Pointer ptr, Word currentHeader) { + if (GraalDirectives.inIntrinsic()) { + ReplacementsUtil.staticAssert(!hasFixedIdentityHashField(), "must always access field"); + } else { + VMError.guarantee(!hasFixedIdentityHashField()); + assert !hasIdentityHashFromAddress(currentHeader); + } + UnsignedWord fromAddressState = IDHASH_STATE_FROM_ADDRESS.shiftLeft(IDHASH_STATE_SHIFT); + UnsignedWord newHeader = currentHeader.and(IDHASH_STATE_BITS.not()).or(fromAddressState); + writeHeaderToObject(ptr.toObjectNonNull(), newHeader); + if (!GraalDirectives.inIntrinsic()) { + assert hasIdentityHashFromAddress(readHeaderFromObject(ptr)); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean hasIdentityHashFromAddress(Word header) { + return hasIdentityHashFromAddressInline(header); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static boolean hasIdentityHashFromAddressInline(Word header) { + if (hasFixedIdentityHashField()) { + return false; } + UnsignedWord fromAddressState = IDHASH_STATE_FROM_ADDRESS.shiftLeft(IDHASH_STATE_SHIFT); + return header.and(IDHASH_STATE_BITS).equal(fromAddressState); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -176,6 +275,7 @@ private static void dynamicAssert(boolean condition, String msg) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void writeHeaderToObject(Object o, WordBase header) { if (getReferenceSize() == Integer.BYTES) { ObjectAccess.writeInt(o, getHubOffset(), (int) header.rawValue()); @@ -190,16 +290,15 @@ public Word encodeAsTLABObjectHeader(DynamicHub hub) { } @Uninterruptible(reason = "Called from uninterruptible code.") - public static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, boolean unaligned) { + public Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, boolean unaligned) { /* * All DynamicHub instances are in the native image heap and therefore do not move, so we * can convert the hub to a Pointer without any precautions. */ Word result = Word.objectToUntrackedPointer(hub); if (SubstrateOptions.SpawnIsolates.getValue()) { - if (hasBase()) { - result = result.subtract(KnownIntrinsics.heapBase()); - } + result = result.subtract(KnownIntrinsics.heapBase()); + result = result.shiftLeft(numReservedExtraBits); } if (rememberedSet) { result = result.or(REMEMBERED_SET_BIT); @@ -212,8 +311,9 @@ public static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, b /** Clear the object header bits from a header. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord clearBits(UnsignedWord header) { - return header.and(CLEAR_HEADER_BITS); + UnsignedWord clearBits(UnsignedWord header) { + UnsignedWord mask = WordFactory.unsigned(reservedBitsMask); + return header.and(mask.not()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -236,8 +336,9 @@ public static boolean isConsumedHeapChunkZapped(UnsignedWord header) { @Override public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFromHeapBase) { - long header = hubOffsetFromHeapBase; - assert (header & MASK_HEADER_BITS.rawValue()) == 0 : "Object header bits must be zero initially"; + long header = hubOffsetFromHeapBase << numReservedExtraBits; + VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); + assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { if (obj.getPartition() instanceof ChunkedImageHeapPartition) { ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); @@ -251,9 +352,13 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro assert obj.getPartition() instanceof FillerObjectDummyPartition; } } + if (!hasFixedIdentityHashField()) { + header |= (IDHASH_STATE_IN_FIELD.rawValue() << IDHASH_STATE_SHIFT); + } return header; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } @@ -262,11 +367,13 @@ public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isUnalignedObject(Object obj) { UnsignedWord header = ObjectHeaderImpl.readHeaderFromObject(obj); return isUnalignedHeader(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } @@ -281,24 +388,22 @@ public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } - public static boolean isPointerToForwardedObject(Pointer p) { - UnsignedWord header = readHeaderFromPointer(p); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + boolean isPointerToForwardedObject(Pointer p) { + Word header = readHeaderFromPointer(p); return isForwardedHeader(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isForwardedHeader(UnsignedWord header) { - return testForwardedHeaderBit(header); + return header.and(FORWARDED_BIT).notEqual(0); } - private static boolean testForwardedHeaderBit(UnsignedWord headerBits) { - return headerBits.and(FORWARDED_BIT).notEqual(0); - } - - static Object getForwardedObject(Pointer ptr) { + Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } - static Object getForwardedObject(Pointer ptr, UnsignedWord header) { + Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { @@ -318,14 +423,14 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") - static void installForwardingPointer(Object original, Object copy) { + void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); + forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); @@ -333,22 +438,16 @@ static void installForwardingPointer(Object original, Object copy) { } else { forwardHeader = Word.objectToUntrackedPointer(copy); } - assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); + assert getHeaderBitsFromHeader(forwardHeader).equal(0); writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static UnsignedWord getHeaderBitsFromHeader(UnsignedWord header) { + private UnsignedWord getHeaderBitsFromHeader(UnsignedWord header) { assert !isProducedHeapChunkZapped(header) : "Produced chunk zap value"; assert !isConsumedHeapChunkZapped(header) : "Consumed chunk zap value"; - return header.and(MASK_HEADER_BITS); - } - - static UnsignedWord getHeaderBitsFromHeaderCarefully(UnsignedWord header) { - VMError.guarantee(!isProducedHeapChunkZapped(header), "Produced chunk zap value"); - VMError.guarantee(!isConsumedHeapChunkZapped(header), "Consumed chunk zap value"); - return header.and(MASK_HEADER_BITS); + return header.and(reservedBitsMask); } @Fold @@ -356,18 +455,13 @@ static int getHubOffset() { return ConfigurationValues.getObjectLayout().getHubOffset(); } - @Fold - static int getIdentityHashCodeOffset() { - return ConfigurationValues.getObjectLayout().getIdentityHashCodeOffset(); - } - @Fold static int getReferenceSize() { return ConfigurationValues.getObjectLayout().getReferenceSize(); } @Fold - static boolean hasBase() { - return ImageSingletons.lookup(CompressEncoding.class).hasBase(); + static boolean hasFixedIdentityHashField() { + return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java index 9b15f461e7d0..58ff2e67c927 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java @@ -121,7 +121,7 @@ private void adjustDesiredTenuringThreshold() { // DefNewGeneration::adjust_desi tenuringThreshold = Math.min(i + 1, MAX_TENURING_THRESHOLD); } - private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // CardGeneration::compute_new_size + private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredGeneration::compute_new_size_inner UnsignedWord capacityAtPrologue = oldSize; UnsignedWord usedAfterGc = GCImpl.getGCImpl().getAccounting().getOldGenerationAfterChunkBytes(); if (oldSize.belowThan(usedAfterGc)) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReciprocalLeastSquareFit.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReciprocalLeastSquareFit.java index 0b2f3efb8ab8..5ed700cc0b54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReciprocalLeastSquareFit.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReciprocalLeastSquareFit.java @@ -45,7 +45,8 @@ final class ReciprocalLeastSquareFit { private double a; private double b; - ReciprocalLeastSquareFit(int effectiveHistoryLength) { + ReciprocalLeastSquareFit(double effectiveHistoryLength) { + assert effectiveHistoryLength > 0; this.discount = (effectiveHistoryLength - 1.0) / effectiveHistoryLength; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index e0bffb409384..429afe8fa0b9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -218,9 +218,10 @@ private static boolean processRememberedRef(Reference dr) { } private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(referentAddr); + ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); + UnsignedWord header = ohi.readHeaderFromPointer(referentAddr); if (ObjectHeaderImpl.isForwardedHeader(header)) { - Object forwardedObj = ObjectHeaderImpl.getForwardedObject(referentAddr); + Object forwardedObj = ohi.getForwardedObject(referentAddr); ReferenceInternals.setReferent(dr, forwardedObj); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java index 33f5a1a978c2..e53e54a5334b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java @@ -24,10 +24,10 @@ */ package com.oracle.svm.core.genscavenge; +import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -68,7 +68,8 @@ public static boolean isReachable(Pointer ptrToObj) { return true; } - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(ptrToObj); + ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); + Word header = ohi.readHeaderFromPointer(ptrToObj); if (ObjectHeaderImpl.isForwardedHeader(header)) { return true; } @@ -78,7 +79,6 @@ public static boolean isReachable(Pointer ptrToObj) { return true; } - ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); Class clazz = DynamicHub.toClass(ohi.dynamicHubFromObjectHeader(header)); return isAssumedReachable(clazz); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java index 1893f634e44c..337b438b0388 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeImageHeapChunkWriter.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.genscavenge.remset.RememberedSet; +import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.image.ImageHeapObject; import sun.nio.ch.DirectBuffer; @@ -59,6 +60,7 @@ public void initializeAlignedChunk(int chunkPosition, long topOffset, long endOf header.setSpace(null); header.setOffsetToPreviousChunk(WordFactory.unsigned(offsetToPreviousChunk)); header.setOffsetToNextChunk(WordFactory.unsigned(offsetToNextChunk)); + header.setIdentityHashSalt(WordFactory.zero(), IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); } @Override @@ -69,6 +71,7 @@ public void initializeUnalignedChunk(int chunkPosition, long topOffset, long end header.setSpace(null); header.setOffsetToPreviousChunk(WordFactory.unsigned(offsetToPreviousChunk)); header.setOffsetToNextChunk(WordFactory.unsigned(offsetToNextChunk)); + header.setIdentityHashSalt(WordFactory.zero(), IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 206eb97a519c..680bcff2c8a4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -24,10 +24,11 @@ */ package com.oracle.svm.core.genscavenge; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.AlwaysInline; +import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -35,14 +36,17 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; @@ -351,7 +355,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { - ObjectHeaderImpl.installForwardingPointer(original, copy); + ObjectHeaderImpl.getObjectHeaderImpl().installForwardingPointer(original, copy); } return copy; } @@ -361,8 +365,18 @@ private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(originalObj); - Pointer copyMemory = allocateMemory(size); + UnsignedWord originalSize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, false); + UnsignedWord copySize = originalSize; + boolean addIdentityHashField = false; + if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { + Word header = ObjectHeaderImpl.readHeaderFromObject(originalObj); + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) { + addIdentityHashField = true; + copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true); + } + } + + Pointer copyMemory = allocateMemory(copySize); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { return null; } @@ -373,9 +387,16 @@ private Object copyAlignedObject(Object originalObj) { * later on anyways (the card table is also updated at that point if necessary). */ Pointer originalMemory = Word.objectToUntrackedPointer(originalObj); - UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, originalSize); Object copy = copyMemory.toObject(); + if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { + // Must do first: ensures correct object size below and in other places + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj); + int offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); + } if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered // set bit and the first object table (even when promoting from old to old). diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index ebaf1818dccb..e04769be8a9b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -236,7 +236,7 @@ private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) { AlignedHeader newTlab = HeapImpl.getChunkProvider().produceAlignedChunk(); return allocateInstanceInNewTlab(hub, newTlab); } finally { - ObjectAllocationInNewTLABEvent.emit(startTicks, hub, LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()), HeapParameters.getAlignedHeapChunkSize()); + ObjectAllocationInNewTLABEvent.emit(startTicks, hub, LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()), HeapParameters.getAlignedHeapChunkSize()); DeoptTester.enableDeoptTesting(); } } @@ -268,7 +268,7 @@ private static Object slowPathNewArrayLikeObject(Word objectHeader, int length, } DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(objectHeader); - UnsignedWord size = LayoutEncoding.getArraySize(hub.getLayoutEncoding(), length); + UnsignedWord size = LayoutEncoding.getArrayAllocationSize(hub.getLayoutEncoding(), length); /* * Check if the array is too big. This is an optimistic check because the heap probably * has other objects in it, and the next collection could throw an OutOfMemoryError if @@ -322,7 +322,7 @@ private static Object slowPathNewArrayLikeObject0(DynamicHub hub, int length, Un @Uninterruptible(reason = "Holds uninitialized memory.") private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader newTlabChunk) { - UnsignedWord size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()); + UnsignedWord size = LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()); Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk); return FormatObjectNode.formatObject(memory, DynamicHub.toClass(hub), false, FillContent.WITH_ZEROES, true); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index dbe9f164a2d4..b6673242aec1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -128,11 +128,13 @@ public static Pointer allocateMemory(UnalignedHeader that, UnsignedWord size) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnalignedHeader getEnclosingChunk(Object obj) { Pointer objPointer = Word.objectToUntrackedPointer(obj); return getEnclosingChunkFromObjectPointer(objPointer); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static UnalignedHeader getEnclosingChunkFromObjectPointer(Pointer objPointer) { Pointer chunkPointer = objPointer.subtract(getObjectStartOffset()); return (UnalignedHeader) chunkPointer; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java index ce475be3449b..73d4402b923a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java @@ -69,7 +69,7 @@ public static Object formatObjectSnippet(Word memory, DynamicHub hub, boolean re @ConstantParameter AllocationSnippets.AllocationSnippetCounters snippetCounters) { DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); int layoutEncoding = hubNonNull.getLayoutEncoding(); - UnsignedWord size = LayoutEncoding.getPureInstanceSize(layoutEncoding); + UnsignedWord size = LayoutEncoding.getPureInstanceAllocationSize(layoutEncoding); Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, false); return alloc().formatObject(objectHeader, size, memory, fillContents, emitMemoryBarrier, false, snippetCounters); } @@ -80,7 +80,7 @@ public static Object formatArraySnippet(Word memory, DynamicHub hub, int length, @ConstantParameter AllocationSnippets.AllocationSnippetCounters snippetCounters) { DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); int layoutEncoding = hubNonNull.getLayoutEncoding(); - UnsignedWord size = LayoutEncoding.getArraySize(layoutEncoding, length); + UnsignedWord size = LayoutEncoding.getArrayAllocationSize(layoutEncoding, length); Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned); return alloc().formatArray(objectHeader, size, length, memory, fillContents, emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters); } @@ -90,7 +90,7 @@ public static Object formatStoredContinuation(Word memory, DynamicHub hub, int l @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter AllocationSnippets.AllocationSnippetCounters snippetCounters) { DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); int layoutEncoding = hubNonNull.getLayoutEncoding(); - UnsignedWord size = LayoutEncoding.getArraySize(layoutEncoding, length); + UnsignedWord size = LayoutEncoding.getArrayAllocationSize(layoutEncoding, length); Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned); return alloc().formatStoredContinuation(objectHeader, size, length, memory, emitMemoryBarrier, ipOffset, snippetCounters); } @@ -103,13 +103,13 @@ public static Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLeng byte[] refMapNonNull = (byte[]) PiNode.piCastNonNull(referenceMap, SnippetAnchorNode.anchor()); Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned); int layoutEncoding = hubNonNull.getLayoutEncoding(); - UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, arrayLength); + UnsignedWord allocationSize = LayoutEncoding.getArrayAllocationSize(layoutEncoding, arrayLength); return alloc().formatPod(objectHeader, hubNonNull, allocationSize, arrayLength, refMapNonNull, memory, fillContents, emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters); } private static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, boolean unaligned) { - return ObjectHeaderImpl.encodeAsObjectHeader(hub, rememberedSet, unaligned); + return ObjectHeaderImpl.getObjectHeaderImpl().encodeAsObjectHeader(hub, rememberedSet, unaligned); } @Fold diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index 890774f41c0e..ed25afb9808d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -85,7 +85,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); Pointer startOffset = Word.objectToUntrackedPointer(obj).subtract(objectsStart); - Pointer endOffset = LayoutEncoding.getObjectEnd(obj).subtract(objectsStart); + Pointer endOffset = LayoutEncoding.getObjectEndInGC(obj).subtract(objectsStart); FirstObjectTable.setTableForObject(fotStart, startOffset, endOffset); ObjectHeaderImpl.setRememberedSetBit(obj); } @@ -101,7 +101,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { while (offset.belowThan(top)) { Object obj = offset.toObject(); enableRememberedSetForObject(chunk, obj); - offset = offset.add(LayoutEncoding.getSizeFromObject(obj)); + offset = offset.add(LayoutEncoding.getSizeFromObjectInGC(obj)); } } @@ -145,7 +145,7 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito while (ptr.belowThan(walkLimit)) { Object obj = ptr.toObject(); visitor.visitObjectInline(obj); - ptr = LayoutEncoding.getObjectEndInline(obj); + ptr = LayoutEncoding.getObjectEndInlineInGC(obj); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index 5bf1c95dd4be..4796a9dde18f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -160,7 +160,7 @@ public static boolean verify(Pointer cardTableStart, Pointer objectsStart, Point success &= verifyReferent(ref, cardTableStart, objectsStart); } } - curPtr = LayoutEncoding.getObjectEnd(obj); + curPtr = LayoutEncoding.getObjectEndInGC(obj); } return success; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index e613414131bd..1183aa2712dc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -239,7 +239,7 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object // If the object starts before the memory for this index, skip over it. if (firstObject.belowThan(indexedMemoryStart)) { Object crossingObject = firstObject.toObject(); - result = LayoutEncoding.getObjectEnd(crossingObject); + result = LayoutEncoding.getObjectEndInGC(crossingObject); } else { assert firstObject.equal(indexedMemoryStart) : "preciseFirstPointer.equal(indexedMemoryStart)"; result = indexedMemoryStart; @@ -308,7 +308,7 @@ public static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer o } Object obj = objStart.toObject(); - Pointer objEnd = LayoutEncoding.getObjectEnd(obj); + Pointer objEnd = LayoutEncoding.getObjectEndInGC(obj); if (!entryStart.belowThan(objEnd)) { Log.log().string("The first object table entry at index ").unsigned(index).string(" points to an object is not crossing nor starting at a card boundary: obj: ").zhex(objStart) .string(" - ").zhex(objEnd).string(", chunk: ").zhex(objectsStart).string(" - ").zhex(objectsLimit).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index 90f7583a49e9..cb8b929c1db0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -91,7 +91,7 @@ private static > T createArray(int length, Class } DynamicHub hub = SubstrateUtil.cast(arrayType, DynamicHub.class); assert LayoutEncoding.isArray(hub.getLayoutEncoding()); - UnsignedWord size = LayoutEncoding.getArraySize(hub.getLayoutEncoding(), length); + UnsignedWord size = LayoutEncoding.getArrayAllocationSize(hub.getLayoutEncoding(), length); Pointer array = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(size); if (array.isNull()) { throw OUT_OF_MEMORY_ERROR; @@ -147,7 +147,7 @@ public static int lengthOf(NonmovableArray array) { /** Provides the size of the given array in bytes. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord byteSizeOf(NonmovableArray array) { - return array.isNonNull() ? LayoutEncoding.getArraySize(readLayoutEncoding(array), lengthOf(array)) : WordFactory.zero(); + return array.isNonNull() ? LayoutEncoding.getArrayAllocationSize(readLayoutEncoding(array), lengthOf(array)) : WordFactory.zero(); } /** @see System#arraycopy */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index 82a7770618c1..febd8215da9b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -24,8 +24,9 @@ */ package com.oracle.svm.core.config; -import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.nativeimage.AnnotationAccess; import org.graalvm.nativeimage.c.constant.CEnum; import org.graalvm.word.WordBase; @@ -53,14 +54,14 @@ public final class ObjectLayout { private final int firstFieldOffset; private final int arrayLengthOffset; private final int arrayBaseOffset; - private final int identityHashCodeOffset; + private final int fixedIdentityHashOffset; - public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int objectAlignment, int hubOffset, int firstFieldOffset, int arrayLengthOffset, int arrayBaseOffset, - int identityHashCodeOffset) { + public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int objectAlignment, int hubOffset, + int firstFieldOffset, int arrayLengthOffset, int arrayBaseOffset, int fixedIdentityHashOffset) { assert CodeUtil.isPowerOf2(referenceSize); assert CodeUtil.isPowerOf2(objectAlignment); assert hubOffset < firstFieldOffset && hubOffset < arrayLengthOffset; - assert identityHashCodeOffset > 0 && identityHashCodeOffset < arrayLengthOffset; + assert fixedIdentityHashOffset == -1 || (fixedIdentityHashOffset > 0 && fixedIdentityHashOffset < arrayLengthOffset); this.target = target; this.referenceSize = referenceSize; @@ -70,7 +71,7 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob this.firstFieldOffset = firstFieldOffset; this.arrayLengthOffset = arrayLengthOffset; this.arrayBaseOffset = arrayBaseOffset; - this.identityHashCodeOffset = identityHashCodeOffset; + this.fixedIdentityHashOffset = fixedIdentityHashOffset; } /** The minimum alignment of objects (instances and arrays). */ @@ -114,6 +115,7 @@ public int getArrayIndexScale(JavaKind kind) { /** * Align the specified offset or address up to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int alignUp(int obj) { return (obj + alignmentMask) & ~alignmentMask; } @@ -121,6 +123,7 @@ public int alignUp(int obj) { /** * Align the specified offset or address up to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long alignUp(long obj) { return (obj + alignmentMask) & ~alignmentMask; } @@ -145,12 +148,24 @@ public int getArrayLengthOffset() { return arrayLengthOffset; } + /** If {@link #hasFixedIdentityHashField()}, then returns that offset. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public int getFixedIdentityHashOffset() { + if (GraalDirectives.inIntrinsic()) { + ReplacementsUtil.dynamicAssert(hasFixedIdentityHashField(), "must check before calling"); + } else { + assert hasFixedIdentityHashField(); + } + return fixedIdentityHashOffset; + } + /** - * Returns the offset of the identity hash code field. + * Indicates whether all objects, including arrays, always contain an identity hash code field + * at a specific offset. */ - @Fold - public int getIdentityHashCodeOffset() { - return identityHashCodeOffset; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean hasFixedIdentityHashField() { + return fixedIdentityHashOffset >= 0; } public int getArrayBaseOffset(JavaKind kind) { @@ -161,21 +176,52 @@ public long getArrayElementOffset(JavaKind kind, int index) { return getArrayBaseOffset(kind) + index * sizeInBytes(kind); } - public long getArraySize(JavaKind kind, int length) { + public long getArraySize(JavaKind kind, int length, boolean withOptionalIdHashField) { + return computeArrayTotalSize(getArrayUnalignedSize(kind, length), withOptionalIdHashField); + } + + private long getArrayUnalignedSize(JavaKind kind, int length) { assert length >= 0; - return alignUp(getArrayBaseOffset(kind) + ((long) length << getArrayIndexShift(kind))); + return getArrayBaseOffset(kind) + ((long) length << getArrayIndexShift(kind)); + } + + public long getArrayOptionalIdentityHashOffset(JavaKind kind, int length) { + return getArrayOptionalIdentityHashOffset(getArrayUnalignedSize(kind, length)); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public long getArrayOptionalIdentityHashOffset(long unalignedSize) { + if (hasFixedIdentityHashField()) { + return getFixedIdentityHashOffset(); + } + int align = Integer.BYTES; + return ((unalignedSize + align - 1) / align) * align; } - public int getMinimumInstanceObjectSize() { - return alignUp(firstFieldOffset); // assumes there are no always-present "synthetic fields" + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public long computeArrayTotalSize(long unalignedSize, boolean withOptionalIdHashField) { + long size = unalignedSize; + if (withOptionalIdHashField && !hasFixedIdentityHashField()) { + size = getArrayOptionalIdentityHashOffset(size) + Integer.BYTES; + } + return alignUp(size); + } + + public int getMinImageHeapInstanceSize() { + int unalignedSize = firstFieldOffset; // assumes no always-present "synthetic fields" + if (!hasFixedIdentityHashField()) { + int idHashOffset = NumUtil.roundUp(unalignedSize, Integer.BYTES); + unalignedSize = idHashOffset + Integer.BYTES; + } + return alignUp(unalignedSize); } - public int getMinimumArraySize() { - return NumUtil.safeToInt(getArraySize(JavaKind.Byte, 0)); + public int getMinImageHeapArraySize() { + return NumUtil.safeToInt(getArraySize(JavaKind.Byte, 0, true)); } - public int getMinimumObjectSize() { - return Math.min(getMinimumArraySize(), getMinimumInstanceObjectSize()); + public int getMinImageHeapObjectSize() { + return Math.min(getMinImageHeapArraySize(), getMinImageHeapInstanceSize()); } public static JavaKind getCallSignatureKind(boolean isEntryPoint, ResolvedJavaType type, MetaAccessProvider metaAccess, TargetDescription target) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java index 7ea0b1eb1df0..7924a0442f89 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java @@ -156,7 +156,7 @@ private static Object doClone(Object original) throws CloneNotSupportedException // copy remaining non-object data int endOffset = isArrayLike ? LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding) - : UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding)); + : UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceAllocationSize(layoutEncoding)); int primitiveDataSize = endOffset - curOffset; assert primitiveDataSize >= 0; assert curOffset >= 0; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SubstrateBasicLoweringProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SubstrateBasicLoweringProvider.java index c1bfa36546c9..6136cdeefe11 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SubstrateBasicLoweringProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SubstrateBasicLoweringProvider.java @@ -47,6 +47,7 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AndNode; +import org.graalvm.compiler.nodes.calc.LeftShiftNode; import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; import org.graalvm.compiler.nodes.extended.LoadHubNode; import org.graalvm.compiler.nodes.extended.LoadMethodNode; @@ -225,16 +226,19 @@ protected ValueNode createReadHub(StructuredGraph graph, ValueNode object, Lower int reservedBitsMask = Heap.getHeap().getObjectHeader().getReservedBitsMask(); if (reservedBitsMask != 0) { // get rid of the reserved header bits and extract the actual pointer to the hub - int encodingShift = ReferenceAccess.singleton().getCompressEncoding().getShift(); - if ((reservedBitsMask >>> encodingShift) == 0) { - hubBits = graph.unique(new UnsignedRightShiftNode(headerBits, ConstantNode.forInt(encodingShift, graph))); - } else if (reservedBitsMask < objectLayout.getAlignment()) { + assert CodeUtil.isPowerOf2(reservedBitsMask + 1) : "only the lowest bits may be set"; + int numReservedBits = CodeUtil.log2(reservedBitsMask + 1); + int compressionShift = ReferenceAccess.singleton().getCompressEncoding().getShift(); + int numAlignmentBits = CodeUtil.log2(objectLayout.getAlignment()); + assert compressionShift <= numAlignmentBits : "compression discards bits"; + if (numReservedBits == numAlignmentBits && compressionShift == 0) { hubBits = graph.unique(new AndNode(headerBits, ConstantNode.forIntegerStamp(headerBitsStamp, ~reservedBitsMask, graph))); } else { - assert encodingShift > 0 : "shift below results in a compressed value"; - assert CodeUtil.isPowerOf2(reservedBitsMask + 1) : "only the lowest bits may be set"; - int numReservedBits = CodeUtil.log2(reservedBitsMask + 1); hubBits = graph.unique(new UnsignedRightShiftNode(headerBits, ConstantNode.forInt(numReservedBits, graph))); + if (compressionShift != numAlignmentBits) { + int shift = numAlignmentBits - compressionShift; + hubBits = graph.unique(new LeftShiftNode(hubBits, ConstantNode.forInt(shift, graph))); + } } } else { hubBits = headerBits; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java index 23a18eb32638..d650ce403710 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java @@ -24,7 +24,7 @@ */ package com.oracle.svm.core.graal.snippets; -import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_LOCATIONS; +import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.GC_LOCATIONS; import java.util.Arrays; import java.util.Map; @@ -79,8 +79,8 @@ private static void safepointSnippet() { } private static LocationIdentity[] getKilledLocations() { - int newLength = TLAB_LOCATIONS.length + 1; - LocationIdentity[] locations = Arrays.copyOf(TLAB_LOCATIONS, newLength); + int newLength = GC_LOCATIONS.length + 1; + LocationIdentity[] locations = Arrays.copyOf(GC_LOCATIONS, newLength); locations[newLength - 1] = Safepoint.getThreadLocalSafepointRequestedLocationIdentity(); return locations; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java index 87002bd73469..26e8f7df3cef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java @@ -94,6 +94,7 @@ import com.oracle.svm.core.heap.Pod; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.meta.SharedType; import com.oracle.svm.core.meta.SubstrateObjectConstant; import com.oracle.svm.core.option.HostedOptionValues; @@ -110,8 +111,9 @@ public class SubstrateAllocationSnippets extends AllocationSnippets { public static final LocationIdentity TLAB_TOP_IDENTITY = NamedLocationIdentity.mutable("TLAB.top"); public static final LocationIdentity TLAB_END_IDENTITY = NamedLocationIdentity.mutable("TLAB.end"); - public static final Object[] ALLOCATION_LOCATIONS = new Object[]{TLAB_TOP_IDENTITY, TLAB_END_IDENTITY, AllocationCounter.COUNT_FIELD, AllocationCounter.SIZE_FIELD}; - public static final LocationIdentity[] TLAB_LOCATIONS = new LocationIdentity[]{TLAB_TOP_IDENTITY, TLAB_END_IDENTITY}; + public static final Object[] ALLOCATION_LOCATIONS = new Object[]{TLAB_TOP_IDENTITY, TLAB_END_IDENTITY, IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION, + AllocationCounter.COUNT_FIELD, AllocationCounter.SIZE_FIELD}; + public static final LocationIdentity[] GC_LOCATIONS = new LocationIdentity[]{TLAB_TOP_IDENTITY, TLAB_END_IDENTITY, IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION}; private static final SubstrateForeignCallDescriptor NEW_MULTI_ARRAY = SnippetRuntime.findForeignCall(SubstrateAllocationSnippets.class, "newMultiArrayStub", true); private static final SubstrateForeignCallDescriptor INSTANCE_HUB_ERROR = SnippetRuntime.findForeignCall(SubstrateAllocationSnippets.class, "instanceHubErrorStub", true); @@ -226,7 +228,7 @@ public Object allocateInstanceDynamic(@NonNullParameter DynamicHub hub, protected Object allocateInstanceDynamicImpl(DynamicHub hub, FillContent fillContents, boolean emitMemoryBarrier, @SuppressWarnings("unused") boolean supportsBulkZeroing, @SuppressWarnings("unused") boolean supportsOptimizedFilling, AllocationProfilingData profilingData) { // The hub was already verified by a ValidateNewInstanceClassNode. - UnsignedWord size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()); + UnsignedWord size = LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()); Object result = allocateInstanceImpl(encodeAsTLABObjectHeader(hub), size, fillContents, emitMemoryBarrier, false, profilingData); return piCastToSnippetReplaceeStamp(result); } @@ -679,7 +681,7 @@ public void lower(NewInstanceNode node, LoweringTool tool) { DynamicHub hub = ensureMarkedAsInstantiated(type.getHub()); ConstantNode hubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(hub), tool.getMetaAccess(), graph); - long size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()).rawValue(); + long size = LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()).rawValue(); Arguments args = new Arguments(allocateInstance, graph.getGuardsStage(), tool.getLoweringStage()); args.add("hub", hubConstant); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ClassHistogramVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ClassHistogramVisitor.java index 0e36675456ff..3ddb8bf09ece 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ClassHistogramVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ClassHistogramVisitor.java @@ -47,7 +47,7 @@ public boolean visitObject(Object o) { return false; } entry.instanceCount++; - entry.instanceSpace += LayoutEncoding.getSizeFromObject(o).rawValue(); + entry.instanceSpace += LayoutEncoding.getMomentarySizeFromObject(o).rawValue(); return true; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index c400b566a974..614af42d1e34 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -40,6 +40,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.PredefinedClassesSupport; +import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.os.CommittedMemoryProvider; @@ -244,4 +245,14 @@ public List> getLoadedClasses() { /** Consider all references in the given object as needing remembered set entries. */ @Uninterruptible(reason = "Ensure that no GC can occur between modification of the object and this call.", callerMustBe = true) public abstract void dirtyAllReferencesOf(Object obj); + + /** + * Retrieves a salt value for computing the {@linkplain System#identityHashCode identity hash + * code} of the passed object (and potentially other objects) from its address. The same salt + * value will be returned for this object at least until the next garbage collection. + * + * Implementations must use {@link IdentityHashCodeSupport#IDENTITY_HASHCODE_SALT_LOCATION}. + */ + @Uninterruptible(reason = "Ensure that no GC can occur between this call and usage of the salt.", callerMustBe = true) + public abstract long getIdentityHashSalt(Object obj); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index cce77abf9fa7..5e83ff7a0932 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -29,7 +29,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.hub.DynamicHub; @@ -62,12 +61,15 @@ protected ObjectHeader() { public abstract Word encodeAsUnmanagedObjectHeader(DynamicHub hub); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public abstract DynamicHub dynamicHubFromObjectHeader(UnsignedWord header); + public abstract DynamicHub dynamicHubFromObjectHeader(Word header); public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public abstract Word readHeaderFromPointer(Pointer ptr); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public abstract DynamicHub readDynamicHubFromPointer(Pointer ptr); @@ -75,7 +77,7 @@ public static DynamicHub readDynamicHubFromObject(Object o) { public abstract Pointer readPotentialDynamicHubFromPointer(Pointer ptr); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public abstract void initializeHeaderOfNewObject(Pointer objectPointer, Word objectHeader); + public abstract void initializeHeaderOfNewObject(Pointer ptr, Word header); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean pointsToObjectHeader(Pointer ptr) { @@ -87,6 +89,15 @@ public boolean pointsToObjectHeader(Pointer ptr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public abstract boolean hasOptionalIdentityHashField(Word header); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public abstract boolean hasIdentityHashFromAddress(Word header); + + @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.", callerMustBe = true) + public abstract void setIdentityHashFromAddress(Pointer ptr, Word currentHeader); + @Fold protected static int getCompressionShift() { return ReferenceAccess.singleton().getCompressEncoding().getShift(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java index 2272d3b830ee..752c9f25e9eb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java @@ -98,7 +98,12 @@ boolean walkHeapObjectsWithoutAllocating(WalkHeapObjectsOperation operation) { public int instanceSizeOf(Class cls) { final int encoding = DynamicHub.fromClass(cls).getLayoutEncoding(); if (LayoutEncoding.isPureInstance(encoding)) { - return (int) LayoutEncoding.getPureInstanceSize(encoding).rawValue(); + /* + * May underestimate the object size if the identity hashcode field is optional. This is + * the best that what can do because the HPROF format does not support that instances of + * one class have different object sizes. + */ + return (int) LayoutEncoding.getPureInstanceAllocationSize(encoding).rawValue(); } else { return 0; } @@ -320,7 +325,7 @@ public long sizeOf(Object obj) { if (obj == null) { result = 0; } else { - final UnsignedWord objectSize = LayoutEncoding.getSizeFromObject(obj); + final UnsignedWord objectSize = LayoutEncoding.getMomentarySizeFromObject(obj); result = objectSize.rawValue(); } return result; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index 3e7b97e52203..11412eb47e76 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -77,6 +77,8 @@ import com.oracle.svm.core.annotate.TargetElement; import com.oracle.svm.core.classinitialization.ClassInitializationInfo; import com.oracle.svm.core.classinitialization.EnsureClassInitializedNode; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.config.ObjectLayout; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.jdk.JDK11OrEarlier; import com.oracle.svm.core.jdk.JDK17OrLater; @@ -173,6 +175,8 @@ public final class DynamicHub implements AnnotatedElement, java.lang.reflect.Typ */ private short monitorOffset; + private short optionalIdentityHashOffset; + /** * Bit-set for various boolean flags, to reduce size of instances. It is important that this * field is final, so that various methods are constant folded for constant classes already @@ -404,14 +408,21 @@ public void setClassInitializationInfo(ClassInitializationInfo classInitializati } @Platforms(Platform.HOSTED_ONLY.class) - public void setData(int layoutEncoding, int typeID, int monitorOffset, short typeCheckStart, short typeCheckRange, short typeCheckSlot, short[] typeCheckSlots, - CFunctionPointer[] vtable, long referenceMapIndex, boolean isInstantiated, boolean canInstantiateAsInstance) { + public void setData(int layoutEncoding, int typeID, int monitorOffset, int optionalIdentityHashOffset, short typeCheckStart, short typeCheckRange, short typeCheckSlot, + short[] typeCheckSlots, CFunctionPointer[] vtable, long referenceMapIndex, boolean isInstantiated, boolean canInstantiateAsInstance) { assert this.vtable == null : "Initialization must be called only once"; assert !(!isInstantiated && canInstantiateAsInstance); + if (LayoutEncoding.isPureInstance(layoutEncoding)) { + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + assert ol.hasFixedIdentityHashField() ? (optionalIdentityHashOffset == ol.getFixedIdentityHashOffset()) : (optionalIdentityHashOffset > 0); + } else { + assert optionalIdentityHashOffset == -1; + } this.layoutEncoding = layoutEncoding; this.typeID = typeID; this.monitorOffset = NumUtil.safeToShort(monitorOffset); + this.optionalIdentityHashOffset = NumUtil.safeToShort(optionalIdentityHashOffset); this.typeCheckStart = typeCheckStart; this.typeCheckRange = typeCheckRange; this.typeCheckSlot = typeCheckSlot; @@ -573,6 +584,15 @@ public int getMonitorOffset() { return monitorOffset; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + int getOptionalIdentityHashOffset() { + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + if (ol.hasFixedIdentityHashField()) { // enable elimination of our field + return ol.getFixedIdentityHashOffset(); + } + return optionalIdentityHashOffset; + } + public DynamicHub getSuperHub() { return superHub; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index eb866e0f3847..1e63276c3a3c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -24,11 +24,9 @@ */ package com.oracle.svm.core.hub; -import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.calc.UnsignedMath; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -37,7 +35,10 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.VMError; @@ -124,7 +125,7 @@ public static int forPureInstance(ResolvedJavaType type, int size) { guaranteeEncoding(type, false, isHybrid(encoding), "Instance type encoding denotes a hybrid"); guaranteeEncoding(type, false, isObjectArray(encoding) || isArrayLikeWithObjectElements(encoding), "Instance type encoding denotes an object array"); guaranteeEncoding(type, false, isPrimitiveArray(encoding) || isArrayLikeWithPrimitiveElements(encoding), "Instance type encoding denotes a primitive array"); - guaranteeEncoding(type, true, getPureInstanceSize(encoding).equal(WordFactory.unsigned(size)), "Instance type encoding size matches type size"); + guaranteeEncoding(type, true, getPureInstanceAllocationSize(encoding).equal(WordFactory.unsigned(size)), "Instance type encoding size matches type size"); return encoding; } @@ -183,12 +184,32 @@ public static boolean isPureInstance(int encoding) { return encoding > LAST_SPECIAL_VALUE; } - /** Determines the size of a pure instance object (i.e. not a hybrid object or array). */ + /** + * Determines the size of a pure instance object (i.e. not a hybrid object or array) at + * allocation, that is, without an identity hash code field if such a field is optional. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getPureInstanceSize(int encoding) { + public static UnsignedWord getPureInstanceAllocationSize(int encoding) { return WordFactory.unsigned(encoding); } + /** + * Determines the size of a pure instance object (i.e. not a hybrid object or array) with or + * without an identity hash code field (if such a field is optional). + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getPureInstanceSize(DynamicHub hub, boolean withOptionalIdHashField) { + UnsignedWord size = getPureInstanceAllocationSize(hub.getLayoutEncoding()); + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + if (withOptionalIdHashField && !ol.hasFixedIdentityHashField()) { + int afterIdHashField = hub.getOptionalIdentityHashOffset() + Integer.BYTES; + if (size.belowThan(afterIdHashField)) { // fits in a gap between fields + size = WordFactory.unsigned(ol.alignUp(afterIdHashField)); + } + } + return size; + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isArray(int encoding) { int mask = (ARRAY_TAG_IDENTITY_BIT | ARRAY_TAG_PURE_BIT) << ARRAY_TAG_SHIFT; @@ -253,38 +274,112 @@ public static UnsignedWord getArrayElementOffset(int encoding, int index) { return getArrayBaseOffset(encoding).add(WordFactory.unsigned(index).shiftLeft(getArrayIndexShift(encoding))); } + /** + * Determines the size of an array at allocation, that is, without an identity hash code field + * if such a field is optional. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getArraySize(int encoding, int length) { - int alignmentMask = getAlignmentMask(); - return getArrayElementOffset(encoding, length).add(alignmentMask).and(~alignmentMask); + public static UnsignedWord getArrayAllocationSize(int encoding, int length) { + return getArraySize(encoding, length, false); } + /** + * Determines the size of an array with or without an identity hash code field, if such a field + * is optional. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getArraySize(int encoding, int length, boolean withOptionalIdHashField) { + long unalignedSize = getArrayElementOffset(encoding, length).rawValue(); + long totalSize = ConfigurationValues.getObjectLayout().computeArrayTotalSize(unalignedSize, withOptionalIdHashField); + return WordFactory.unsigned(totalSize); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int getOptionalIdentityHashOffset(Object obj) { + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + if (ol.hasFixedIdentityHashField()) { + return ol.getFixedIdentityHashOffset(); + } + DynamicHub hub = KnownIntrinsics.readHub(obj); + int encoding = hub.getLayoutEncoding(); + if (isArrayLike(encoding)) { + long unalignedSize = getArrayElementOffset(encoding, ArrayLengthNode.arrayLength(obj)).rawValue(); + return (int) ol.getArrayOptionalIdentityHashOffset(unalignedSize); + } else { + return hub.getOptionalIdentityHashOffset(); + } + } + + @Uninterruptible(reason = "Prevent a GC moving the object or interfering with its identity hash state.", callerMustBe = true) public static UnsignedWord getSizeFromObject(Object obj) { - return getSizeFromObjectInline(obj); + boolean withOptionalIdHashField = !ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj); + return getSizeFromObjectInline(obj, withOptionalIdHashField); + } + + public static UnsignedWord getSizeFromObjectAddOptionalIdHashField(Object obj) { + return getSizeFromObjectInline(obj, true); + } + + /** + * Returns the size of the object in the instant of the call, which can have already become + * stale after returning. This can be useful for diagnostic output. + */ + @Uninterruptible(reason = "Caller is aware the value can be stale, but required by callee.") + public static UnsignedWord getMomentarySizeFromObject(Object obj) { + return getSizeFromObject(obj); + } + + public static UnsignedWord getSizeFromObjectInGC(Object obj) { + return getSizeFromObjectInlineInGC(obj); + } + + @AlwaysInline("GC performance") + public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { + return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { + boolean withOptionalIdHashField = addOptionalIdHashField || + (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); + return getSizeFromObjectInline(obj, withOptionalIdHashField); + } + + @AlwaysInline("Actual inlining decided by callers.") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromObjectInline(Object obj) { - int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding(); + private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOptionalIdHashField) { + DynamicHub hub = KnownIntrinsics.readHub(obj); + int encoding = hub.getLayoutEncoding(); if (isArrayLike(encoding)) { - return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); + return getArraySize(encoding, ArrayLengthNode.arrayLength(obj), withOptionalIdHashField); } else { - return getPureInstanceSize(encoding); + return getPureInstanceSize(hub, withOptionalIdHashField); } } - /** Returns the end of the Object when the call started, e.g., for logging. */ - public static Pointer getObjectEnd(Object obj) { - return getObjectEndInline(obj); + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean checkOptionalIdentityHashField(Object obj) { + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + Word header = oh.readHeaderFromPointer(Word.objectToUntrackedPointer(obj)); + return oh.hasOptionalIdentityHashField(header); + } + + public static Pointer getObjectEndInGC(Object obj) { + return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") - public static Pointer getObjectEndInline(Object obj) { - final Pointer objStart = Word.objectToUntrackedPointer(obj); - final UnsignedWord objSize = getSizeFromObjectInline(obj); - return objStart.add(objSize); + public static Pointer getObjectEndInlineInGC(Object obj) { + UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); + return Word.objectToUntrackedPointer(obj).add(size); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer getImageHeapObjectEnd(Object obj) { + // Image heap objects never move and always have an identity hash code field. + UnsignedWord size = getSizeFromObjectInline(obj, true); + return Word.objectToUntrackedPointer(obj).add(size); } public static boolean isArray(Object obj) { @@ -296,9 +391,4 @@ public static boolean isArrayLike(Object obj) { final int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding(); return isArrayLike(encoding); } - - @Fold - protected static int getAlignmentMask() { - return ImageSingletons.lookup(ObjectLayout.class).getAlignment() - 1; - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java index d2d2a573a466..32ffd9585564 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java @@ -31,9 +31,15 @@ import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.IdentityHashCodeSnippets; import org.graalvm.compiler.word.ObjectAccess; +import org.graalvm.compiler.word.Word; import org.graalvm.word.LocationIdentity; +import org.graalvm.word.SignedWord; +import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalObject; @@ -44,6 +50,13 @@ public final class IdentityHashCodeSupport { public static final LocationIdentity IDENTITY_HASHCODE_LOCATION = NamedLocationIdentity.mutable("identityHashCode"); + /** + * Location representing the {@linkplain Heap#getIdentityHashSalt salt values used for the + * identity hash code of objects}. These values change between collections, so this location + * must be killed at safepoint checks and allocation slow-paths. + */ + public static final LocationIdentity IDENTITY_HASHCODE_SALT_LOCATION = NamedLocationIdentity.mutable("identityHashCodeSalt"); + private static final FastThreadLocalObject hashCodeGeneratorTL = FastThreadLocalFactory.createObject(SplittableRandom.class, "IdentityHashCodeSupport.hashCodeGeneratorTL"); /** @@ -55,22 +68,39 @@ public static void ensureInitialized() { } public static IdentityHashCodeSnippets.Templates createSnippetTemplates(OptionValues options, Providers providers) { - return new IdentityHashCodeSnippets.Templates(new SubstrateIdentityHashCodeSnippets(), options, providers, IDENTITY_HASHCODE_LOCATION); + return SubstrateIdentityHashCodeSnippets.createTemplates(options, providers); } @SubstrateForeignCallTarget(stubCallingConvention = false) public static int generateIdentityHashCode(Object obj) { - - // generate a new hashcode and try to store it into the object - int newHashCode = generateHashCode(); - if (!Unsafe.getUnsafe().compareAndSetInt(obj, ConfigurationValues.getObjectLayout().getIdentityHashCodeOffset(), 0, newHashCode)) { - newHashCode = ObjectAccess.readInt(obj, ConfigurationValues.getObjectLayout().getIdentityHashCodeOffset(), IDENTITY_HASHCODE_LOCATION); + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + VMError.guarantee(ol.hasFixedIdentityHashField(), "Snippet must handle other cases"); + int newHashCode = generateRandomHashCode(); + if (!Unsafe.getUnsafe().compareAndSetInt(obj, ol.getFixedIdentityHashOffset(), 0, newHashCode)) { + newHashCode = ObjectAccess.readInt(obj, ol.getFixedIdentityHashOffset(), IDENTITY_HASHCODE_LOCATION); } VMError.guarantee(newHashCode != 0, "Missing identity hash code"); return newHashCode; } - private static int generateHashCode() { + @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.") + public static int computeHashCodeFromAddress(Object obj) { + Word address = Word.objectToUntrackedPointer(obj); + long salt = Heap.getHeap().getIdentityHashSalt(obj); + SignedWord salted = WordFactory.signed(salt).xor(address); + int hash = mix32(salted.rawValue()) >>> 1; // shift: ensure positive, same as on HotSpot + return (hash == 0) ? 1 : hash; // ensure nonzero + } + + /** Avalanching bit mixer, from {@link SplittableRandom}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static int mix32(long a) { + long z = a; + z = (z ^ (z >>> 33)) * 0x62a9d9ed799705f5L; + return (int) (((z ^ (z >>> 28)) * 0xcb24d0a5c88c35b3L) >>> 32); + } + + private static int generateRandomHashCode() { SplittableRandom hashCodeGenerator = hashCodeGeneratorTL.get(); if (hashCodeGenerator == null) { /* diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeNode.java index 43114508e585..1c598f60d9e7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeNode.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeNode.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.identityhashcode; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.type.TypedConstant; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeCycles; @@ -33,9 +34,12 @@ import org.graalvm.compiler.replacements.nodes.IdentityHashCodeNode; import org.graalvm.word.LocationIdentity; +import com.oracle.svm.core.config.ConfigurationValues; + import jdk.vm.ci.meta.JavaConstant; -@NodeInfo(cycles = NodeCycles.CYCLES_2, size = NodeSize.SIZE_8) +@NodeInfo(cycles = NodeCycles.CYCLES_UNKNOWN, cyclesRationale = "Decided depending on identity hash code storage.", // + size = NodeSize.SIZE_UNKNOWN, sizeRationale = "Decided depending on identity hash code storage.") public final class SubstrateIdentityHashCodeNode extends IdentityHashCodeNode { public static final NodeClass TYPE = NodeClass.create(SubstrateIdentityHashCodeNode.class); @@ -54,11 +58,27 @@ protected SubstrateIdentityHashCodeNode(ValueNode object, int bci) { @Override public LocationIdentity getKilledLocationIdentity() { - return IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION; + // Without a fixed field, we must write bits in the object header. + return haveFixedField() ? IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION : LocationIdentity.any(); } @Override protected int getIdentityHashCode(JavaConstant constant) { return ((TypedConstant) constant).getIdentityHashCode(); } + + @Override + public NodeCycles estimatedNodeCycles() { + return haveFixedField() ? NodeCycles.CYCLES_2 : NodeCycles.CYCLES_8; + } + + @Override + protected NodeSize dynamicNodeSizeEstimate() { + return haveFixedField() ? NodeSize.SIZE_8 : NodeSize.SIZE_32; + } + + @Fold + static boolean haveFixedField() { + return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeSnippets.java index 72c6db8aa747..d384d0c4faab 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/SubstrateIdentityHashCodeSnippets.java @@ -24,33 +24,64 @@ */ package com.oracle.svm.core.identityhashcode; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.IdentityHashCodeSnippets; import org.graalvm.compiler.word.ObjectAccess; +import org.graalvm.compiler.word.Word; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.ObjectHeader; +import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.snippets.SnippetRuntime; import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor; final class SubstrateIdentityHashCodeSnippets extends IdentityHashCodeSnippets { - static final SubstrateForeignCallDescriptor GENERATE_IDENTITY_HASH_CODE = SnippetRuntime.findForeignCall(IdentityHashCodeSupport.class, "generateIdentityHashCode", true, - IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + static final SubstrateForeignCallDescriptor GENERATE_IDENTITY_HASH_CODE = SnippetRuntime.findForeignCall( + IdentityHashCodeSupport.class, "generateIdentityHashCode", true, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + + static Templates createTemplates(OptionValues options, Providers providers) { + return new Templates(new SubstrateIdentityHashCodeSnippets(), options, providers, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + } @Override protected int computeIdentityHashCode(Object obj) { - int identityHashCode = ObjectAccess.readInt(obj, ConfigurationValues.getObjectLayout().getIdentityHashCodeOffset(), IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); - if (probability(FAST_PATH_PROBABILITY, identityHashCode != 0)) { + int identityHashCode; + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + if (ol.hasFixedIdentityHashField()) { + int offset = ol.getFixedIdentityHashOffset(); + identityHashCode = ObjectAccess.readInt(obj, offset, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + if (probability(SLOW_PATH_PROBABILITY, identityHashCode == 0)) { + identityHashCode = generateIdentityHashCode(GENERATE_IDENTITY_HASH_CODE, obj); + } return identityHashCode; + } + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + Word objPtr = Word.objectToUntrackedPointer(obj); + Word header = oh.readHeaderFromPointer(objPtr); + if (probability(LIKELY_PROBABILITY, oh.hasOptionalIdentityHashField(header))) { + int offset = LayoutEncoding.getOptionalIdentityHashOffset(obj); + identityHashCode = ObjectAccess.readInt(obj, offset, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); } else { - return generateIdentityHashCode(GENERATE_IDENTITY_HASH_CODE, obj); + identityHashCode = IdentityHashCodeSupport.computeHashCodeFromAddress(obj); + if (probability(NOT_FREQUENT_PROBABILITY, !oh.hasIdentityHashFromAddress(header))) { + // Note this write leads to frame state issues that break scheduling if done earlier + oh.setIdentityHashFromAddress(objPtr, header); + } } + return identityHashCode; } @NodeIntrinsic(ForeignCallNode.class) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/SnippetRuntime.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/SnippetRuntime.java index 380e0e1711e2..cfdf31450b7d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/SnippetRuntime.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/snippets/SnippetRuntime.java @@ -24,7 +24,7 @@ */ package com.oracle.svm.core.snippets; -import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_LOCATIONS; +import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.GC_LOCATIONS; import java.lang.reflect.Method; @@ -102,14 +102,14 @@ private static SubstrateForeignCallDescriptor findForeignCall(String descriptorN if (isFullyUninterruptible) { VMError.guarantee(isUninterruptible, "%s is fully uninterruptible but not annotated with @Uninterruptible.", method); killedLocations = additionalKilledLocations; - } else if (additionalKilledLocations.length == 0 || additionalKilledLocations == TLAB_LOCATIONS) { - killedLocations = TLAB_LOCATIONS; + } else if (additionalKilledLocations.length == 0 || additionalKilledLocations == GC_LOCATIONS) { + killedLocations = GC_LOCATIONS; } else if (containsAny(additionalKilledLocations)) { killedLocations = additionalKilledLocations; } else { - killedLocations = new LocationIdentity[TLAB_LOCATIONS.length + additionalKilledLocations.length]; - System.arraycopy(TLAB_LOCATIONS, 0, killedLocations, 0, TLAB_LOCATIONS.length); - System.arraycopy(additionalKilledLocations, 0, killedLocations, TLAB_LOCATIONS.length, additionalKilledLocations.length); + killedLocations = new LocationIdentity[GC_LOCATIONS.length + additionalKilledLocations.length]; + System.arraycopy(GC_LOCATIONS, 0, killedLocations, 0, GC_LOCATIONS.length); + System.arraycopy(additionalKilledLocations, 0, killedLocations, GC_LOCATIONS.length, additionalKilledLocations.length); } boolean needsDebugInfo = !isFullyUninterruptible; diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMemoryAccessProviderImpl.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMemoryAccessProviderImpl.java index 8f5b5cbb26ec..644fb6ff908b 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMemoryAccessProviderImpl.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMemoryAccessProviderImpl.java @@ -150,7 +150,7 @@ private static JavaConstant readPrimitiveChecked(JavaKind kind, Constant baseCon } else if (displacement <= 0) { /* Trying to read before the object, or the hub. No need to look into the object. */ return null; - } else if (WordFactory.unsigned(displacement + bits / 8).aboveThan(LayoutEncoding.getSizeFromObject(baseObject))) { + } else if (WordFactory.unsigned(displacement + bits / 8).aboveThan(LayoutEncoding.getMomentarySizeFromObject(baseObject))) { /* Trying to read after the end of the object. */ return null; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index cb00c3089e24..0fcd6d512828 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -107,40 +107,61 @@ public static void setDefaultIfEmpty() { } public static ObjectLayout createObjectLayout() { - return createObjectLayout(JavaKind.Object); + return createObjectLayout(JavaKind.Object, false); } /** - * Defines the layout of objects. + * Defines the layout of objects. Identity hash code fields can be optional if the object header + * allows for it, in which case such a field is appended to individual objects after an identity + * hash code has been assigned to it (unless there is an otherwise unused gap in the object that + * can be used). * * The layout of instance objects is: *
    - *
  • hub (reference)
    • - *
  • identity hashcode (int)
    • + *
  • object header with hub reference
    • + *
  • optional: identity hashcode (int)
    • *
  • instance fields (references, primitives)
    • *
  • if needed, object monitor (reference)
    • *
* * The layout of array objects is: *
    - *
  • hub (reference)
    • - *
  • identity hashcode (int)
    • + *
  • object header with hub reference
    • + *
  • optional: identity hashcode (int)
    • *
  • array length (int)
    • *
  • array elements (length * reference or primitive)
    • *
*/ - public static ObjectLayout createObjectLayout(JavaKind referenceKind) { + public static ObjectLayout createObjectLayout(JavaKind referenceKind, boolean disableOptionalIdentityHash) { SubstrateTargetDescription target = ConfigurationValues.getTarget(); int referenceSize = target.arch.getPlatformKind(referenceKind).getSizeInBytes(); + int headerSize = referenceSize; + int intSize = target.arch.getPlatformKind(JavaKind.Int).getSizeInBytes(); int objectAlignment = 8; - int hubOffset = 0; - int identityHashCodeOffset = hubOffset + referenceSize; - int firstFieldOffset = identityHashCodeOffset + target.arch.getPlatformKind(JavaKind.Int).getSizeInBytes(); + int headerOffset = 0; + int identityHashCodeOffset; + int firstFieldOffset; + if (!disableOptionalIdentityHash && SubstrateOptions.SpawnIsolates.getValue() && headerSize + referenceSize <= objectAlignment) { + /* + * References are relative to the heap base, so we should be able to use fewer bits in + * the object header to reference DynamicHubs which are located near the start of the + * heap. This means we could be unable to fit forwarding references in those header bits + * during GC, but every object is large enough to fit a separate forwarding reference + * outside its header. Therefore, we can avoid reserving an identity hash code field for + * every object during its allocation and use extra header bits to track if an + * individual object was assigned an identity hash code after allocation. + */ + identityHashCodeOffset = -1; + firstFieldOffset = headerOffset + headerSize; + } else { // need all object header bits except for lowest-order bits freed up by alignment + identityHashCodeOffset = headerOffset + referenceSize; + firstFieldOffset = identityHashCodeOffset + intSize; + } int arrayLengthOffset = firstFieldOffset; - int arrayBaseOffset = arrayLengthOffset + target.arch.getPlatformKind(JavaKind.Int).getSizeInBytes(); + int arrayBaseOffset = arrayLengthOffset + intSize; - return new ObjectLayout(target, referenceSize, objectAlignment, hubOffset, firstFieldOffset, arrayLengthOffset, arrayBaseOffset, identityHashCodeOffset); + return new ObjectLayout(target, referenceSize, objectAlignment, headerOffset, firstFieldOffset, arrayLengthOffset, arrayBaseOffset, identityHashCodeOffset); } public SVMHost createHostVM(OptionValues options, ClassLoader classLoader, ClassInitializationSupport classInitializationSupport, diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index f22d8c843f41..49511ddc8de5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -1769,7 +1769,7 @@ private void printTypes() { } else if (LayoutEncoding.isAbstract(le)) { System.out.print("abstract "); } else if (LayoutEncoding.isPureInstance(le)) { - System.out.format("instance size %d ", LayoutEncoding.getPureInstanceSize(le).rawValue()); + System.out.format("instance size %d ", LayoutEncoding.getPureInstanceAllocationSize(le).rawValue()); } else if (LayoutEncoding.isArrayLike(le)) { String arrayType = LayoutEncoding.isHybrid(le) ? "hybrid" : "array"; String elements = LayoutEncoding.isArrayLikeWithPrimitiveElements(le) ? "primitives" : "objects"; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java index 8228b75aacf0..39625a95c1ba 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java @@ -95,8 +95,12 @@ public long getArrayElementOffset(int index) { return getArrayBaseOffset() + ((long) index) * layout.sizeInBytes(getArrayElementStorageKind()); } - public long getTotalSize(int length) { - return layout.alignUp(getArrayElementOffset(length)); + public long getTotalSize(int length, boolean withOptionalIdHashField) { + return layout.computeArrayTotalSize(getArrayElementOffset(length), withOptionalIdHashField); + } + + public long getOptionalIdentityHashOffset(int length) { + return layout.getArrayOptionalIdentityHashOffset(getArrayElementOffset(length)); } public HostedField getArrayField() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageDebugInfoProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageDebugInfoProvider.java index 08f36fea5a9a..3bea5e2279d4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageDebugInfoProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageDebugInfoProvider.java @@ -642,16 +642,21 @@ private Stream computeHeaderTypeInfo() { List infos = new LinkedList<>(); int hubOffset = getObjectLayout().getHubOffset(); int hubFieldSize = referenceSize; - int idHashOffset = getObjectLayout().getIdentityHashCodeOffset(); + int objHeaderSize = hubOffset + hubFieldSize; + int idHashSize = getObjectLayout().sizeInBytes(JavaKind.Int); - int objHeaderSize = getObjectLayout().getMinimumInstanceObjectSize(); + int fixedIdHashOffset = -1; + if (getObjectLayout().hasFixedIdentityHashField()) { + fixedIdHashOffset = getObjectLayout().getFixedIdentityHashOffset(); + objHeaderSize = Math.max(objHeaderSize, fixedIdHashOffset + idHashSize); + } /* We need array headers for all Java kinds */ NativeImageHeaderTypeInfo objHeader = new NativeImageHeaderTypeInfo("_objhdr", objHeaderSize); objHeader.addField("hub", hubType, hubOffset, hubFieldSize); - if (idHashOffset > 0) { - objHeader.addField("idHash", javaKindToHostedType.get(JavaKind.Int), idHashOffset, idHashSize); + if (fixedIdHashOffset >= 0) { + objHeader.addField("idHash", javaKindToHostedType.get(JavaKind.Int), fixedIdHashOffset, idHashSize); } infos.add(objHeader); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java index 3c48d720ec90..1fb1c6ad4705 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java @@ -138,12 +138,13 @@ public NativeImageHeap(AnalysisUniverse aUniverse, HostedUniverse universe, Host this.objectLayout = ConfigurationValues.getObjectLayout(); this.heapLayouter = heapLayouter; - this.minInstanceSize = objectLayout.getMinimumInstanceObjectSize(); - this.minArraySize = objectLayout.getMinimumArraySize(); + this.minInstanceSize = objectLayout.getMinImageHeapInstanceSize(); + this.minArraySize = objectLayout.getMinImageHeapArraySize(); + assert assertFillerObjectSizes(); + if (ImageHeapConnectedComponentsFeature.Options.PrintImageHeapConnectedComponents.getValue()) { this.objectReachabilityInfo = new IdentityHashMap<>(); } - assert assertFillerObjectSizes(); } @Override @@ -378,7 +379,7 @@ public ObjectInfo addFillerObject(int size) { if (size >= minArraySize) { int elementSize = objectLayout.getArrayIndexScale(JavaKind.Int); int arrayLength = (size - minArraySize) / elementSize; - assert objectLayout.getArraySize(JavaKind.Int, arrayLength) == size; + assert objectLayout.getArraySize(JavaKind.Int, arrayLength, true) == size; return addLateToImageHeap(new int[arrayLength], HeapInclusionReason.FillerObject); } else if (size >= minInstanceSize) { return addLateToImageHeap(new FillerObject(), HeapInclusionReason.FillerObject); @@ -388,10 +389,10 @@ public ObjectInfo addFillerObject(int size) { } private boolean assertFillerObjectSizes() { - assert minArraySize == objectLayout.getArraySize(JavaKind.Int, 0); + assert minArraySize == objectLayout.getArraySize(JavaKind.Int, 0, true); HostedType filler = metaAccess.lookupJavaType(FillerObject.class); - UnsignedWord fillerSize = LayoutEncoding.getPureInstanceSize(filler.getHub().getLayoutEncoding()); + UnsignedWord fillerSize = LayoutEncoding.getPureInstanceSize(filler.getHub(), true); assert fillerSize.equal(minInstanceSize); assert minInstanceSize * 2 >= minArraySize : "otherwise, we might need more than one non-array object"; @@ -476,9 +477,9 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable } assert hybridArray != null : "Cannot read value for field " + hybridArrayField.format("%H.%n"); - size = hybridLayout.getTotalSize(Array.getLength(hybridArray)); + size = hybridLayout.getTotalSize(Array.getLength(hybridArray), true); } else { - size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()).rawValue(); + size = LayoutEncoding.getPureInstanceSize(hub, true).rawValue(); } info = addToImageHeap(constant, clazz, size, identityHashCode, reason); @@ -522,7 +523,7 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable } else if (type.isArray()) { HostedArrayClass clazz = (HostedArrayClass) type; int length = universe.getConstantReflectionProvider().readArrayLength(constant); - final long size = objectLayout.getArraySize(type.getComponentType().getStorageKind(), length); + final long size = objectLayout.getArraySize(type.getComponentType().getStorageKind(), length, true); info = addToImageHeap(constant, clazz, size, identityHashCode, reason); try { recursiveAddObject(hub, false, info); @@ -636,9 +637,9 @@ private long getSize(Object object, HostedType type) { if (type.isInstanceClass()) { HostedInstanceClass clazz = (HostedInstanceClass) type; assert !HybridLayout.isHybrid(clazz); - return LayoutEncoding.getPureInstanceSize(clazz.getHub().getLayoutEncoding()).rawValue(); + return LayoutEncoding.getPureInstanceSize(clazz.getHub(), true).rawValue(); } else if (type.isArray()) { - return objectLayout.getArraySize(type.getComponentType().getStorageKind(), Array.getLength(object)); + return objectLayout.getArraySize(type.getComponentType().getStorageKind(), Array.getLength(object), true); } else { throw shouldNotReachHere(); } @@ -724,10 +725,6 @@ public final class ObjectInfo implements ImageHeapObject { */ private final Object reason; - ObjectInfo(Object object, long size, HostedClass clazz, int identityHashCode, Object reason) { - this(SubstrateObjectConstant.forObject(object), size, clazz, identityHashCode, reason); - } - ObjectInfo(JavaConstant constant, long size, HostedClass clazz, int identityHashCode, Object reason) { this.constant = constant; this.clazz = clazz; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java index 023f361dcae2..2f3d015df53b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java @@ -58,6 +58,7 @@ import com.oracle.svm.hosted.image.NativeImageHeap.ObjectInfo; import com.oracle.svm.hosted.meta.HostedClass; import com.oracle.svm.hosted.meta.HostedField; +import com.oracle.svm.hosted.meta.HostedInstanceClass; import com.oracle.svm.hosted.meta.MaterializedConstantFields; import jdk.internal.misc.Unsafe; @@ -360,7 +361,7 @@ private void writeObject(ObjectInfo info, RelocatableBuffer buffer) { writeField(buffer, info, field, con, info); } } - bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getIdentityHashCodeOffset()), info.getIdentityHashCode()); + long idHashOffset; if (hybridArray != null) { /* * Write the hybrid array length and the array elements. @@ -373,7 +374,11 @@ private void writeObject(ObjectInfo info, RelocatableBuffer buffer) { final Object array = Array.get(hybridArray, i); writeConstant(buffer, elementIndex, elementStorageKind, array, info); } + idHashOffset = hybridLayout.getOptionalIdentityHashOffset(length); + } else { + idHashOffset = ((HostedInstanceClass) clazz).getOptionalIdentityHashOffset(); } + bufferBytes.putInt(info.getIndexInBuffer(idHashOffset), info.getIdentityHashCode()); } else if (clazz.isArray()) { @@ -384,7 +389,7 @@ private void writeObject(ObjectInfo info, RelocatableBuffer buffer) { AnalysisConstantReflectionProvider constantReflection = heap.getUniverse().getConstantReflectionProvider(); int length = constantReflection.readArrayLength(constant); bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getArrayLengthOffset()), length); - bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getIdentityHashCodeOffset()), info.getIdentityHashCode()); + bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getArrayOptionalIdentityHashOffset(kind, length)), info.getIdentityHashCode()); constantReflection.forEachArrayElement(constant, (element, index) -> { final int elementIndex = info.getIndexInBuffer(objectLayout.getArrayElementOffset(kind, index)); writeConstant(buffer, elementIndex, kind, element, info); @@ -396,7 +401,7 @@ private void writeObject(ObjectInfo info, RelocatableBuffer buffer) { Object array = info.getObject(); int length = Array.getLength(array); bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getArrayLengthOffset()), length); - bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getIdentityHashCodeOffset()), info.getIdentityHashCode()); + bufferBytes.putInt(info.getIndexInBuffer(objectLayout.getArrayOptionalIdentityHashOffset(kind, length)), info.getIdentityHashCode()); if (array instanceof Object[]) { Object[] oarray = (Object[]) array; assert oarray.length == length; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedInstanceClass.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedInstanceClass.java index 782442bcb3a3..7ecc8c2664fa 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedInstanceClass.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedInstanceClass.java @@ -37,6 +37,7 @@ public class HostedInstanceClass extends HostedClass { protected int instanceSize; protected boolean monitorFieldNeeded = false; protected int monitorFieldOffset = 0; + protected int optionalIdentityHashOffset = -1; public HostedInstanceClass(HostedUniverse universe, AnalysisType wrapped, JavaKind kind, JavaKind storageKind, HostedClass superClass, HostedInterface[] interfaces) { super(universe, wrapped, kind, storageKind, superClass, interfaces); @@ -94,7 +95,7 @@ public int getInstanceSize() { } /* - * Monitor field. + * Synthetic fields. */ public boolean needMonitorField() { @@ -113,4 +114,14 @@ public void setMonitorFieldOffset(int monitorFieldOffset) { assert this.monitorFieldOffset == 0 : "setting monitor field offset twice"; this.monitorFieldOffset = monitorFieldOffset; } + + public int getOptionalIdentityHashOffset() { + return optionalIdentityHashOffset; + } + + public void setOptionalIdentityHashOffset(int offset) { + assert this.optionalIdentityHashOffset == -1 : "setting identity hashcode field offset more than once"; + assert offset >= 0; + this.optionalIdentityHashOffset = offset; + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java index 8fc259a625e9..9069ca65dd67 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java @@ -531,8 +531,27 @@ private void layoutInstanceFields(HostedInstanceClass clazz, HostedField[] super */ allFields.sort(FIELD_LOCATION_COMPARATOR); + int sizeWithoutIdHashField = usedBytes.length(); + + // Identity hash code + if (!clazz.isAbstract() && !HybridLayout.isHybrid(clazz)) { + int offset; + if (layout.hasFixedIdentityHashField()) { + offset = layout.getFixedIdentityHashOffset(); + } else { // optional: place in gap if any, or append on demand during GC + int size = Integer.BYTES; + int endOffset = usedBytes.length(); + offset = findGapForField(usedBytes, 0, size, endOffset); + if (offset == -1) { + offset = endOffset + getAlignmentAdjustment(endOffset, size); + } + reserve(usedBytes, offset, size); + } + clazz.setOptionalIdentityHashOffset(offset); + } + clazz.instanceFieldsWithoutSuper = allFields.toArray(new HostedField[0]); - clazz.afterFieldsOffset = usedBytes.length(); + clazz.afterFieldsOffset = sizeWithoutIdHashField; clazz.instanceSize = layout.alignUp(clazz.afterFieldsOffset); if (clazz.instanceFieldsWithoutSuper.length == 0) { @@ -1002,6 +1021,7 @@ private void buildHubs() { int layoutHelper; boolean canInstantiateAsInstance = false; int monitorOffset = 0; + int optionalIdHashOffset = -1; if (type.isInstanceClass()) { HostedInstanceClass instanceClass = (HostedInstanceClass) type; if (instanceClass.isAbstract()) { @@ -1017,6 +1037,7 @@ private void buildHubs() { canInstantiateAsInstance = type.isInstantiated(); } monitorOffset = instanceClass.getMonitorFieldOffset(); + optionalIdHashOffset = instanceClass.getOptionalIdentityHashOffset(); } else if (type.isArray()) { JavaKind storageKind = type.getComponentType().getStorageKind(); boolean isObject = (storageKind == JavaKind.Object); @@ -1049,8 +1070,8 @@ private void buildHubs() { long referenceMapIndex = referenceMapEncoder.lookupEncoding(referenceMap); DynamicHub hub = type.getHub(); - hub.setData(layoutHelper, type.getTypeID(), monitorOffset, type.getTypeCheckStart(), type.getTypeCheckRange(), type.getTypeCheckSlot(), - type.getTypeCheckSlots(), vtable, referenceMapIndex, type.isInstantiated(), canInstantiateAsInstance); + hub.setData(layoutHelper, type.getTypeID(), monitorOffset, optionalIdHashOffset, type.getTypeCheckStart(), type.getTypeCheckRange(), + type.getTypeCheckSlot(), type.getTypeCheckSlots(), vtable, referenceMapIndex, type.isInstantiated(), canInstantiateAsInstance); } } diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshake.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshake.java index 87a85b515dfa..875ea0cf9099 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshake.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshake.java @@ -24,7 +24,7 @@ */ package com.oracle.svm.truffle.api; -import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_LOCATIONS; +import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.GC_LOCATIONS; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; @@ -61,7 +61,7 @@ public final class SubstrateThreadLocalHandshake extends ThreadLocalHandshake { - public static final SubstrateForeignCallDescriptor FOREIGN_POLL = SnippetRuntime.findForeignCall(SubstrateThreadLocalHandshake.class, "pollStub", true, TLAB_LOCATIONS); + public static final SubstrateForeignCallDescriptor FOREIGN_POLL = SnippetRuntime.findForeignCall(SubstrateThreadLocalHandshake.class, "pollStub", true, GC_LOCATIONS); static final SubstrateThreadLocalHandshake SINGLETON = new SubstrateThreadLocalHandshake(); diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java index 2b0ae5e55dd7..9b7626eccb8c 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java @@ -24,7 +24,7 @@ */ package com.oracle.svm.truffle.api; -import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_LOCATIONS; +import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.GC_LOCATIONS; import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; import java.util.Arrays; @@ -77,8 +77,8 @@ public SubstrateThreadLocalHandshakeSnippets(OptionValues options, Providers pro } private static LocationIdentity[] getPollKilledLocations() { - int newLength = TLAB_LOCATIONS.length + 1; - LocationIdentity[] locations = Arrays.copyOf(TLAB_LOCATIONS, newLength); + int newLength = GC_LOCATIONS.length + 1; + LocationIdentity[] locations = Arrays.copyOf(GC_LOCATIONS, newLength); locations[newLength - 1] = SubstrateThreadLocalHandshake.PENDING.getLocationIdentity(); return locations; }