diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerBuildingSupport.java index 6b2e134a9bef..31cd3e41c2b8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerBuildingSupport.java @@ -73,6 +73,11 @@ private static ImageLayerBuildingSupport singleton() { return ImageSingletons.lookup(ImageLayerBuildingSupport.class); } + @Platforms(Platform.HOSTED_ONLY.class) + public static boolean firstImageBuild() { + return !buildingImageLayer() || buildingInitialLayer(); + } + @Platforms(Platform.HOSTED_ONLY.class) public static boolean lastImageBuild() { return !buildingImageLayer() || buildingApplicationLayer(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/FeatureSingleton.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/FeatureSingleton.java index 99ee84654129..c527e60e4fe0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/FeatureSingleton.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/FeatureSingleton.java @@ -27,9 +27,10 @@ import java.util.EnumSet; /** - * Feature singletons are hosted only and can only be accessed during build time. + * Feature singletons are hosted only and can only be accessed during build time. Further, we + * currently do not allow features to save information across layers. */ -public interface FeatureSingleton extends LayeredImageSingleton { +public interface FeatureSingleton extends UnsavedSingleton { @Override default EnumSet getImageBuilderFlags() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/InitialLayerOnlyImageSingleton.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/InitialLayerOnlyImageSingleton.java new file mode 100644 index 000000000000..4f566aa842ea --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/layeredimagesingleton/InitialLayerOnlyImageSingleton.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.layeredimagesingleton; + +import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.util.VMError; + +public interface InitialLayerOnlyImageSingleton extends LayeredImageSingleton { + + @Override + default PersistFlags preparePersist(ImageSingletonWriter writer) { + VMError.guarantee(ImageLayerBuildingSupport.buildingInitialLayer(), "This singleton should only be installed in the initial layer"); + return PersistFlags.FORBIDDEN; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 03a374d4f949..d62a822e78de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -67,7 +67,7 @@ import com.oracle.svm.core.threadlocal.FastThreadLocal; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalInt; -import com.oracle.svm.core.threadlocal.VMThreadLocalInfos; +import com.oracle.svm.core.threadlocal.VMThreadLocalOffsetProvider; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.VMError; @@ -405,7 +405,7 @@ public static LocationIdentity getThreadLocalSafepointRequestedLocationIdentity( } public static int getThreadLocalSafepointRequestedOffset() { - return VMThreadLocalInfos.getOffset(safepointRequested); + return VMThreadLocalOffsetProvider.getOffset(safepointRequested); } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalInfos.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalInfos.java index 85a54b8b0db4..1566f30757ad 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalInfos.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalInfos.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import java.util.Collection; +import java.util.EnumSet; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; @@ -42,12 +43,14 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.layeredimagesingleton.InitialLayerOnlyImageSingleton; +import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingletonBuilderFlags; import com.oracle.svm.core.log.Log; import jdk.graal.compiler.word.Word; @AutomaticallyRegisteredImageSingleton -public class VMThreadLocalInfos { +public class VMThreadLocalInfos implements InitialLayerOnlyImageSingleton { /** * The {@link VMThreadLocalInfo} objects are scanned during analysis as soon as they are * discovered. After analysis, they are sorted and stored in the infos field. @@ -127,4 +130,9 @@ public static int getOffset(FastThreadLocal threadLocal) { } return -1; } + + @Override + public EnumSet getImageBuilderFlags() { + return LayeredImageSingletonBuilderFlags.ALL_ACCESS; + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalOffsetProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalOffsetProvider.java new file mode 100644 index 000000000000..f33e0d26b422 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalOffsetProvider.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.threadlocal; + +import org.graalvm.nativeimage.ImageSingletons; + +import com.oracle.svm.core.SubstrateUtil; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +public interface VMThreadLocalOffsetProvider { + + static int getOffset(FastThreadLocal threadLocal) { + if (SubstrateUtil.HOSTED) { + return ImageSingletons.lookup(VMThreadLocalOffsetProvider.class).offsetOf(threadLocal); + } else { + return VMThreadLocalInfos.getOffset(threadLocal); + } + + } + + @Platforms(Platform.HOSTED_ONLY.class) + int offsetOf(FastThreadLocal threadLocal); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java index 4ab6e7c8a2a2..3c9b63e801f2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.threadlocal; +import java.util.EnumSet; + import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -38,10 +40,12 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.heap.UnknownPrimitiveField; +import com.oracle.svm.core.layeredimagesingleton.InitialLayerOnlyImageSingleton; +import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingletonBuilderFlags; import jdk.graal.compiler.api.replacements.Fold; -public class VMThreadLocalSupport { +public class VMThreadLocalSupport implements InitialLayerOnlyImageSingleton { @UnknownPrimitiveField(availability = ReadyForCompilation.class) public int vmThreadSize = -1; @UnknownObjectField(availability = ReadyForCompilation.class) public byte[] vmThreadReferenceMapEncoding; @UnknownPrimitiveField(availability = ReadyForCompilation.class) public long vmThreadReferenceMapIndex = -1; @@ -74,4 +78,9 @@ public void walk(IsolateThread isolateThread, ObjectReferenceVisitor referenceVi NonmovableArray threadRefMapEncoding = NonmovableArrays.fromImageHeap(vmThreadReferenceMapEncoding); InstanceReferenceMapDecoder.walkOffsetsFromPointer((Pointer) isolateThread, threadRefMapEncoding, vmThreadReferenceMapIndex, referenceVisitor, null); } + + @Override + public EnumSet getImageBuilderFlags() { + return LayeredImageSingletonBuilderFlags.ALL_ACCESS; + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/LayeredVMThreadLocalCollector.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/LayeredVMThreadLocalCollector.java new file mode 100644 index 000000000000..9975c70bff3d --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/LayeredVMThreadLocalCollector.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.thread; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.layeredimagesingleton.ImageSingletonLoader; +import com.oracle.svm.core.layeredimagesingleton.ImageSingletonWriter; +import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingleton; +import com.oracle.svm.core.threadlocal.FastThreadLocal; +import com.oracle.svm.core.threadlocal.VMThreadLocalInfo; +import com.oracle.svm.core.threadlocal.VMThreadLocalInfos; +import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.debug.Assertions; + +/** + * When building layered images we assume all used thread locals will be seen within the base layer. + * Subsequent layers then need to be assigned the same offset. In addition, we match based on the + * name assigned to ThreadLocals, so this must be unique. + * + * Note it is possible to relax this constraint of only allowing thread locals to be defined in the + * initial layer. However, doing so will require adjusting {@link VMThreadLocalInfos} to be a + * multi-layered singleton and also {@link VMThreadLocalSupport} to likely be an application layer + * only image singleton. + */ +public class LayeredVMThreadLocalCollector extends VMThreadLocalCollector implements LayeredImageSingleton { + + record ThreadInfo(int size, int offset) { + + } + + final Map threadLocalAssignmentMap; + private final boolean initialLayer; + private int nextOffset; + + public LayeredVMThreadLocalCollector() { + this(null, -1); + } + + private LayeredVMThreadLocalCollector(Map threadLocalAssignmentMap, int nextOffset) { + super(true); + + this.threadLocalAssignmentMap = threadLocalAssignmentMap; + initialLayer = ImageLayerBuildingSupport.buildingInitialLayer(); + this.nextOffset = nextOffset; + } + + @Override + public Object apply(Object source) { + /* + * Make sure all names have been assigned in the prior layers + */ + if (!initialLayer) { + if (source instanceof FastThreadLocal threadLocal) { + var name = threadLocal.getName(); + VMError.guarantee(threadLocalAssignmentMap.containsKey(name), "Found thread local which was not created in the initial layer %s", name); + } + } + return super.apply(source); + } + + @Override + public int sortAndAssignOffsets() { + if (initialLayer) { + assert nextOffset == -1 : nextOffset; + + nextOffset = super.sortAndAssignOffsets(); + } else { + assert nextOffset != -1; + + for (VMThreadLocalInfo info : threadLocals.values()) { + var assignment = threadLocalAssignmentMap.get(info.name); + info.offset = assignment.offset(); + assert assignment.size() == calculateSize(info) : Assertions.errorMessage("Mismatch in computed size: ", assignment.size(), calculateSize(info), info.name); + info.sizeInBytes = assignment.size(); + } + } + + return nextOffset; + } + + @Override + public int getOffset(FastThreadLocal threadLocal) { + if (initialLayer) { + return super.getOffset(threadLocal); + } else { + return threadLocalAssignmentMap.get(threadLocal.getName()).offset(); + } + } + + @Override + public PersistFlags preparePersist(ImageSingletonWriter writer) { + /* + * Store the (name, offset, size) tuple of all thread locals. + */ + List threadLocalNames = new ArrayList<>(); + List threadLocalOffsets = new ArrayList<>(); + List threadLocalSizes = new ArrayList<>(); + if (initialLayer) { + for (var threadLocal : getSortedThreadLocalInfos()) { + threadLocalNames.add(threadLocal.name); + threadLocalOffsets.add(threadLocal.offset); + threadLocalSizes.add(threadLocal.sizeInBytes); + } + } else { + for (var entry : threadLocalAssignmentMap.entrySet()) { + threadLocalNames.add(entry.getKey()); + threadLocalOffsets.add(entry.getValue().offset()); + threadLocalSizes.add(entry.getValue().size()); + } + } + + writer.writeStringList("threadLocalNames", threadLocalNames); + writer.writeIntList("threadLocalOffsets", threadLocalOffsets); + writer.writeIntList("threadLocalSizes", threadLocalSizes); + + /* + * Note while it is not strictly necessary to store nextOffset at the moment, if in the + * future we allow multiple layers to define thread locals then this information will need + * to be propagated. + */ + writer.writeInt("nextOffset", nextOffset); + return PersistFlags.CREATE; + } + + @SuppressWarnings("unused") + public static Object createFromLoader(ImageSingletonLoader loader) { + /* + * Load the (name, offset, size) tuple of all thread locals. + */ + HashMap threadLocalAssignmentMap = new HashMap<>(); + Iterator threadLocalNames = loader.readStringList("threadLocalNames").iterator(); + Iterator threadLocalOffsets = loader.readIntList("threadLocalOffsets").iterator(); + Iterator threadLocalSizes = loader.readIntList("threadLocalSizes").iterator(); + + while (threadLocalNames.hasNext()) { + String threadLocalName = threadLocalNames.next(); + int threadLocalOffset = threadLocalOffsets.next(); + int threadLocalSize = threadLocalSizes.next(); + + var previous = threadLocalAssignmentMap.put(threadLocalName, new ThreadInfo(threadLocalSize, threadLocalOffset)); + assert previous == null : previous; + } + + return new LayeredVMThreadLocalCollector(Map.copyOf(threadLocalAssignmentMap), loader.readInt("nextOffset")); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadFeature.java index aaf5a3b22612..30efae276d8b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadFeature.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.hosted.thread; -import java.util.List; - import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platforms; @@ -41,17 +39,17 @@ import com.oracle.svm.core.graal.thread.LoadVMThreadLocalNode; import com.oracle.svm.core.graal.thread.StoreVMThreadLocalNode; import com.oracle.svm.core.heap.InstanceReferenceMapEncoder; -import com.oracle.svm.core.heap.SubstrateReferenceMap; +import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.layeredimagesingleton.FeatureSingleton; import com.oracle.svm.core.threadlocal.FastThreadLocal; import com.oracle.svm.core.threadlocal.FastThreadLocalBytes; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.threadlocal.VMThreadLocalInfo; import com.oracle.svm.core.threadlocal.VMThreadLocalInfos; +import com.oracle.svm.core.threadlocal.VMThreadLocalOffsetProvider; import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; -import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.nodes.ReadReservedRegister; -import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.core.common.memory.BarrierType; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; import jdk.graal.compiler.nodes.ValueNode; @@ -69,14 +67,21 @@ */ @AutomaticallyRegisteredFeature @Platforms(InternalPlatform.NATIVE_ONLY.class) -public class VMThreadFeature implements InternalFeature { +public class VMThreadFeature implements InternalFeature, VMThreadLocalOffsetProvider, FeatureSingleton { - private final VMThreadLocalCollector threadLocalCollector = new VMThreadLocalCollector(); - private final VMThreadLocalSupport threadLocalSupport = new VMThreadLocalSupport(); + private VMThreadLocalCollector threadLocalCollector; @Override public void duringSetup(DuringSetupAccess config) { - ImageSingletons.add(VMThreadLocalSupport.class, threadLocalSupport); + + if (ImageLayerBuildingSupport.firstImageBuild()) { + var collector = ImageLayerBuildingSupport.buildingImageLayer() ? new LayeredVMThreadLocalCollector() : new VMThreadLocalCollector(); + ImageSingletons.add(VMThreadLocalCollector.class, collector); + ImageSingletons.add(VMThreadLocalSupport.class, new VMThreadLocalSupport()); + } + + threadLocalCollector = ImageSingletons.lookup(VMThreadLocalCollector.class); + threadLocalCollector.installThreadLocalMap(); /* * While technically threadLocalCollector does not replace an object, it does collect * information needed by the invocation plugin used to create VMThreadLocalAccess nodes (and @@ -219,36 +224,33 @@ private boolean handleGetAddress(GraphBuilderContext b, ResolvedJavaMethod targe @Override public void beforeCompilation(BeforeCompilationAccess config) { - List sortedThreadLocalInfos = threadLocalCollector.sortThreadLocals(); - SubstrateReferenceMap referenceMap = new SubstrateReferenceMap(); - int nextOffset = 0; - for (VMThreadLocalInfo info : sortedThreadLocalInfos) { - int alignment = Math.min(8, info.sizeInBytes); - nextOffset = NumUtil.roundUp(nextOffset, alignment); - - if (info.isObject) { - referenceMap.markReferenceAtOffset(nextOffset, true); - } - info.offset = nextOffset; - nextOffset += info.sizeInBytes; + ImageSingletons.add(VMThreadLocalOffsetProvider.class, this); + int nextOffset = threadLocalCollector.sortAndAssignOffsets(); - if (info.offset > info.maxOffset) { - VMError.shouldNotReachHere("Too many thread local variables with maximum offset " + info.maxOffset + " defined"); - } - } + if (ImageLayerBuildingSupport.firstImageBuild()) { + /* + * This information is installed always in the first image. In subsequent images we only + * need to relay to thread local accesses the previously assigned offsets + */ + + var referenceMap = threadLocalCollector.getReferenceMap(); + InstanceReferenceMapEncoder encoder = new InstanceReferenceMapEncoder(); + encoder.add(referenceMap); + NonmovableArray referenceMapEncoding = encoder.encodeAll(); + + var threadLocalSupport = ImageSingletons.lookup(VMThreadLocalSupport.class); - InstanceReferenceMapEncoder encoder = new InstanceReferenceMapEncoder(); - encoder.add(referenceMap); - NonmovableArray referenceMapEncoding = encoder.encodeAll(); - threadLocalSupport.vmThreadReferenceMapEncoding = NonmovableArrays.getHostedArray(referenceMapEncoding); - threadLocalSupport.vmThreadReferenceMapIndex = encoder.lookupEncoding(referenceMap); - threadLocalSupport.vmThreadSize = nextOffset; + threadLocalSupport.vmThreadReferenceMapEncoding = NonmovableArrays.getHostedArray(referenceMapEncoding); + threadLocalSupport.vmThreadReferenceMapIndex = encoder.lookupEncoding(referenceMap); + threadLocalSupport.vmThreadSize = nextOffset; - /* Remember the final sorted list. */ - VMThreadLocalInfos.setInfos(sortedThreadLocalInfos); + /* Remember the final sorted list. */ + VMThreadLocalInfos.setInfos(threadLocalCollector.getSortedThreadLocalInfos()); + } } + @Override public int offsetOf(FastThreadLocal threadLocal) { - return threadLocalCollector.getInfo(threadLocal).offset; + return threadLocalCollector.getOffset(threadLocal); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadLocalCollector.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadLocalCollector.java index 1123333a2c91..28d26cc2229d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadLocalCollector.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/VMThreadLocalCollector.java @@ -27,36 +27,72 @@ import static com.oracle.svm.core.util.VMError.shouldNotReachHere; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.heap.SubstrateReferenceMap; +import com.oracle.svm.core.layeredimagesingleton.ImageSingletonWriter; +import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingleton; +import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingletonBuilderFlags; +import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.threadlocal.FastThreadLocal; import com.oracle.svm.core.threadlocal.VMThreadLocalInfo; import com.oracle.svm.core.util.ObservableImageHeapMapProvider; +import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.nodes.PiNode; import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.graal.compiler.options.Option; /** * Collects all {@link FastThreadLocal} instances that are actually used by the application. */ -public class VMThreadLocalCollector implements Function { +public class VMThreadLocalCollector implements Function, LayeredImageSingleton { - final Map threadLocals = ObservableImageHeapMapProvider.create(); + public static class Options { + @Option(help = "Ensure all create ThreadLocals have unique names")// + public static final HostedOptionKey ValidateUniqueThreadLocalNames = new HostedOptionKey<>(false); + } + + Map threadLocals; private boolean sealed; + final boolean validateUniqueNames; + final Set seenNames; + + public VMThreadLocalCollector() { + this(false); + } + + protected VMThreadLocalCollector(boolean validateUniqueNames) { + this.validateUniqueNames = validateUniqueNames || Options.ValidateUniqueThreadLocalNames.getValue(); + seenNames = validateUniqueNames ? ConcurrentHashMap.newKeySet() : null; + } + + public void installThreadLocalMap() { + assert threadLocals == null : threadLocals; + threadLocals = ObservableImageHeapMapProvider.create(); + } @Override public Object apply(Object source) { - if (source instanceof FastThreadLocal) { - FastThreadLocal threadLocal = (FastThreadLocal) source; + if (source instanceof FastThreadLocal threadLocal) { if (sealed) { assert threadLocals.containsKey(threadLocal) : "VMThreadLocal must have been discovered during static analysis"; } else { - threadLocals.putIfAbsent(threadLocal, new VMThreadLocalInfo(threadLocal)); + var previous = threadLocals.putIfAbsent(threadLocal, new VMThreadLocalInfo(threadLocal)); + if (previous == null && validateUniqueNames) { + /* + * Ensure this name is unique. + */ + VMError.guarantee(seenNames.add(threadLocal.getName()), "Two VMThreadLocals have the same name: %s", threadLocal.getName()); + } } } /* @@ -66,10 +102,9 @@ public Object apply(Object source) { return source; } - public VMThreadLocalInfo getInfo(FastThreadLocal threadLocal) { + public int getOffset(FastThreadLocal threadLocal) { VMThreadLocalInfo result = threadLocals.get(threadLocal); - assert result != null; - return result; + return result.offset; } public VMThreadLocalInfo findInfo(GraphBuilderContext b, ValueNode threadLocalNode) { @@ -83,22 +118,63 @@ public VMThreadLocalInfo findInfo(GraphBuilderContext b, ValueNode threadLocalNo return result; } - public List sortThreadLocals() { + protected static int calculateSize(VMThreadLocalInfo info) { + if (info.sizeSupplier != null) { + int unalignedSize = info.sizeSupplier.getAsInt(); + assert unalignedSize > 0; + return NumUtil.roundUp(unalignedSize, 8); + } else { + return ConfigurationValues.getObjectLayout().sizeInBytes(info.storageKind); + } + } + + private List sortedThreadLocalInfos; + private SubstrateReferenceMap referenceMap; + + public void sortThreadLocals() { + assert sortedThreadLocalInfos == null && referenceMap == null; + sealed = true; for (VMThreadLocalInfo info : threadLocals.values()) { assert info.sizeInBytes == -1; - if (info.sizeSupplier != null) { - int unalignedSize = info.sizeSupplier.getAsInt(); - assert unalignedSize > 0; - info.sizeInBytes = NumUtil.roundUp(unalignedSize, 8); - } else { - info.sizeInBytes = ConfigurationValues.getObjectLayout().sizeInBytes(info.storageKind); + info.sizeInBytes = calculateSize(info); + } + + sortedThreadLocalInfos = new ArrayList<>(threadLocals.values()); + sortedThreadLocalInfos.sort(VMThreadLocalCollector::compareThreadLocal); + } + + public int sortAndAssignOffsets() { + sortThreadLocals(); + + referenceMap = new SubstrateReferenceMap(); + int nextOffset = 0; + for (VMThreadLocalInfo info : sortedThreadLocalInfos) { + int alignment = Math.min(8, info.sizeInBytes); + nextOffset = NumUtil.roundUp(nextOffset, alignment); + + if (info.isObject) { + referenceMap.markReferenceAtOffset(nextOffset, true); + } + info.offset = nextOffset; + nextOffset += info.sizeInBytes; + + if (info.offset > info.maxOffset) { + VMError.shouldNotReachHere("Too many thread local variables with maximum offset " + info.maxOffset + " defined"); } } - List sortedThreadLocals = new ArrayList<>(threadLocals.values()); - sortedThreadLocals.sort(VMThreadLocalCollector::compareThreadLocal); - return sortedThreadLocals; + return nextOffset; + } + + public SubstrateReferenceMap getReferenceMap() { + assert referenceMap != null; + return referenceMap; + } + + public List getSortedThreadLocalInfos() { + assert sortedThreadLocalInfos != null; + return sortedThreadLocalInfos; } private static int compareThreadLocal(VMThreadLocalInfo info1, VMThreadLocalInfo info2) { @@ -133,4 +209,14 @@ private static ValueNode unPi(ValueNode n) { } return cur; } + + @Override + public final EnumSet getImageBuilderFlags() { + return LayeredImageSingletonBuilderFlags.BUILDTIME_ACCESS_ONLY; + } + + @Override + public PersistFlags preparePersist(ImageSingletonWriter writer) { + return PersistFlags.NOTHING; + } }