Skip to content

Commit a9eb62c

Browse files
peter-hoferpeter-hofer
committed
Allocate StoredContinuation as array via paths specific to Serial GC.
1 parent 3d46b66 commit a9eb62c

File tree

10 files changed

+107
-116
lines changed

10 files changed

+107
-116
lines changed

‎compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/AllocationSnippets.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ protected Object allocateInstanceImpl(Word hub,
6363
result = formatObject(hub, prototypeMarkWord, size, top, fillContents, emitMemoryBarrier, constantSize, profilingData.snippetCounters);
6464
} else {
6565
profilingData.snippetCounters.stub.inc();
66-
result = callNewInstanceStub(hub, size);
66+
result = callNewInstanceStub(hub);
6767
}
6868
profileAllocation(profilingData, size);
6969
return verifyOop(result);
@@ -353,11 +353,6 @@ public void emitPrefetchAllocate(Word address, boolean isArray) {
353353

354354
public abstract void initializeObjectHeader(Word memory, Word hub, Word prototypeMarkWord, boolean isArray);
355355

356-
@SuppressWarnings("unused")
357-
protected Object callNewInstanceStub(Word hub, UnsignedWord size) {
358-
return callNewInstanceStub(hub);
359-
}
360-
361356
protected abstract Object callNewInstanceStub(Word hub);
362357

363358
protected abstract Object callNewArrayStub(Word hub, int length, int fillStartOffset);

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_END_IDENTITY;
2828
import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_TOP_IDENTITY;
2929

30-
import com.oracle.svm.core.SubstrateGCOptions;
31-
import com.oracle.svm.core.threadlocal.FastThreadLocalWord;
3230
import org.graalvm.compiler.api.replacements.Fold;
3331
import org.graalvm.compiler.replacements.AllocationSnippets.FillContent;
3432
import org.graalvm.compiler.word.Word;
@@ -46,6 +44,7 @@
4644
import org.graalvm.word.UnsignedWord;
4745
import org.graalvm.word.WordFactory;
4846

47+
import com.oracle.svm.core.SubstrateGCOptions;
4948
import com.oracle.svm.core.SubstrateOptions;
5049
import com.oracle.svm.core.annotate.RestrictHeapAccess;
5150
import com.oracle.svm.core.annotate.Uninterruptible;
@@ -65,6 +64,7 @@
6564
import com.oracle.svm.core.threadlocal.FastThreadLocal;
6665
import com.oracle.svm.core.threadlocal.FastThreadLocalBytes;
6766
import com.oracle.svm.core.threadlocal.FastThreadLocalFactory;
67+
import com.oracle.svm.core.threadlocal.FastThreadLocalWord;
6868
import com.oracle.svm.core.util.VMError;
6969

7070
/**
@@ -157,7 +157,7 @@ private static Descriptor getTlab() {
157157
}
158158

159159
@SubstrateForeignCallTarget(stubCallingConvention = false)
160-
private static Object slowPathNewInstance(Word objectHeader, UnsignedWord size) {
160+
private static Object slowPathNewInstance(Word objectHeader) {
161161
/*
162162
* Avoid stack overflow errors while producing memory chunks, because that could leave the
163163
* heap in an inconsistent state.
@@ -166,9 +166,7 @@ private static Object slowPathNewInstance(Word objectHeader, UnsignedWord size)
166166
try {
167167
DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(objectHeader);
168168

169-
// the instance either is a frame instance or the size can be read from the hub
170-
assert hub.isStoredContinuationClass() || size.equal(hub.getLayoutEncoding());
171-
Object result = slowPathNewInstanceWithoutAllocating(hub, size);
169+
Object result = slowPathNewInstanceWithoutAllocating(hub);
172170
runSlowPathHooks();
173171
return result;
174172
} finally {
@@ -188,14 +186,14 @@ private static void runSlowPathHooks() {
188186
}
189187

190188
@RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
191-
private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub, UnsignedWord size) {
189+
private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) {
192190
DeoptTester.disableDeoptTesting();
193191
try {
194192
HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.allocateNewInstance", DynamicHub.toClass(hub).getName());
195193
GCImpl.getGCImpl().maybeCollectOnAllocation();
196194

197195
AlignedHeader newTlab = HeapImpl.getChunkProvider().produceAlignedChunk();
198-
return allocateInstanceInNewTlab(hub, size, newTlab);
196+
return allocateInstanceInNewTlab(hub, newTlab);
199197
} finally {
200198
DeoptTester.enableDeoptTesting();
201199
}
@@ -262,9 +260,10 @@ private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int leng
262260
}
263261

264262
@Uninterruptible(reason = "Holds uninitialized memory.")
265-
private static Object allocateInstanceInNewTlab(DynamicHub hub, UnsignedWord size, AlignedHeader newTlabChunk) {
263+
private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader newTlabChunk) {
264+
UnsignedWord size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding());
266265
Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk);
267-
return FormatObjectNode.formatObject(memory, DynamicHub.toClass(hub), size, false, FillContent.WITH_ZEROES, true);
266+
return FormatObjectNode.formatObject(memory, DynamicHub.toClass(hub), false, FillContent.WITH_ZEROES, true);
268267
}
269268

270269
@Uninterruptible(reason = "Holds uninitialized memory.")

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import com.oracle.svm.core.genscavenge.remset.RememberedSet;
6161
import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
6262
import com.oracle.svm.core.graal.snippets.SubstrateTemplates;
63+
import com.oracle.svm.core.heap.StoredContinuation;
6364
import com.oracle.svm.core.option.HostedOptionKey;
6465
import com.oracle.svm.core.util.Counter;
6566
import com.oracle.svm.core.util.CounterFeature;
@@ -88,7 +89,7 @@ static BarrierSnippetCounters counters() {
8889
}
8990

9091
public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>> lowerings) {
91-
PostWriteBarrierLowering lowering = new PostWriteBarrierLowering();
92+
PostWriteBarrierLowering lowering = new PostWriteBarrierLowering(providers);
9293
lowerings.put(SerialWriteBarrier.class, lowering);
9394
// write barriers are currently always imprecise
9495
lowerings.put(SerialArrayRangeWriteBarrier.class, lowering);
@@ -133,6 +134,11 @@ public static void postWriteBarrierSnippet(Object object, @ConstantParameter boo
133134

134135
private class PostWriteBarrierLowering implements NodeLoweringProvider<WriteBarrier> {
135136
private final SnippetInfo postWriteBarrierSnippet = snippet(BarrierSnippets.class, "postWriteBarrierSnippet", CARD_REMEMBERED_SET_LOCATION);
137+
private final ResolvedJavaType storedContinuationType;
138+
139+
PostWriteBarrierLowering(Providers providers) {
140+
storedContinuationType = providers.getMetaAccess().lookupJavaType(StoredContinuation.class);
141+
}
136142

137143
@Override
138144
public void lower(WriteBarrier barrier, LoweringTool tool) {
@@ -148,6 +154,7 @@ public void lower(WriteBarrier barrier, LoweringTool tool) {
148154
* For simplicity, we exclude all interface types.
149155
*/
150156
ResolvedJavaType baseType = StampTool.typeOrNull(address.getBase());
157+
assert baseType == null || !storedContinuationType.isAssignableFrom(baseType) : "StoredContinuation should be effectively immutable and references only be written by GC";
151158
boolean alwaysAlignedChunk = baseType != null && !baseType.isArray() && !baseType.isJavaLangObject() && !baseType.isInterface();
152159

153160
args.add("object", address.getBase());

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java‎

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,21 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge.graal;
2626

27+
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
28+
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
29+
2730
import java.util.Map;
2831

2932
import org.graalvm.compiler.api.replacements.Snippet;
3033
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
3134
import org.graalvm.compiler.graph.Node;
35+
import org.graalvm.compiler.nodes.ConstantNode;
3236
import org.graalvm.compiler.nodes.PiNode;
3337
import org.graalvm.compiler.nodes.SnippetAnchorNode;
3438
import org.graalvm.compiler.nodes.StructuredGraph;
3539
import org.graalvm.compiler.nodes.spi.LoweringTool;
3640
import org.graalvm.compiler.options.OptionValues;
3741
import org.graalvm.compiler.phases.util.Providers;
38-
import org.graalvm.compiler.replacements.ReplacementsUtil;
3942
import org.graalvm.compiler.replacements.SnippetCounter;
4043
import org.graalvm.compiler.replacements.SnippetTemplate;
4144
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
@@ -52,12 +55,16 @@
5255
import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
5356
import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
5457
import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
58+
import com.oracle.svm.core.graal.nodes.NewStoredContinuationNode;
5559
import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
5660
import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
5761
import com.oracle.svm.core.heap.Heap;
5862
import com.oracle.svm.core.heap.StoredContinuation;
63+
import com.oracle.svm.core.heap.StoredContinuationImpl;
5964
import com.oracle.svm.core.hub.DynamicHub;
6065
import com.oracle.svm.core.hub.LayoutEncoding;
66+
import com.oracle.svm.core.meta.SharedType;
67+
import com.oracle.svm.core.meta.SubstrateObjectConstant;
6168
import com.oracle.svm.core.snippets.SnippetRuntime;
6269
import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
6370
import com.oracle.svm.core.thread.Continuation;
@@ -81,18 +88,13 @@ public static void registerLowering(OptionValues options, Providers providers,
8188
}
8289

8390
@Snippet
84-
public Object formatObjectSnippet(Word memory, DynamicHub hub, UnsignedWord size, boolean rememberedSet, FillContent fillContents,
85-
boolean emitMemoryBarrier, @ConstantParameter AllocationSnippetCounters snippetCounters) {
91+
public Object formatObjectSnippet(Word memory, DynamicHub hub, boolean rememberedSet, FillContent fillContents, boolean emitMemoryBarrier,
92+
@ConstantParameter AllocationSnippetCounters snippetCounters) {
8693
DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
94+
int layoutEncoding = hubNonNull.getLayoutEncoding();
95+
UnsignedWord size = LayoutEncoding.getInstanceSize(layoutEncoding);
8796
Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, false);
88-
Object obj = formatObject(objectHeader, WordFactory.nullPointer(), size, memory, fillContents, false, false, snippetCounters);
89-
if (Continuation.isSupported() && hub == DynamicHub.fromClass(StoredContinuation.class)) {
90-
finishFormatStoredContinuation(obj, size.rawValue());
91-
} else {
92-
ReplacementsUtil.dynamicAssert(size.equal(LayoutEncoding.getInstanceSize(hubNonNull.getLayoutEncoding())), "unexpected size");
93-
}
94-
emitMemoryBarrierIf(emitMemoryBarrier);
95-
return obj;
97+
return formatObject(objectHeader, WordFactory.nullPointer(), size, memory, fillContents, emitMemoryBarrier, false, snippetCounters);
9698
}
9799

98100
@Snippet
@@ -102,14 +104,33 @@ public Object formatArraySnippet(Word memory, DynamicHub hub, int length, boolea
102104
int layoutEncoding = hubNonNull.getLayoutEncoding();
103105
UnsignedWord size = LayoutEncoding.getArraySize(layoutEncoding, length);
104106
Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned);
105-
return formatArray(objectHeader, WordFactory.nullPointer(), size, length, memory, fillContents, fillStartOffset,
106-
emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
107+
Object obj = formatArray(objectHeader, WordFactory.nullPointer(), size, length, memory, fillContents, fillStartOffset,
108+
false, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
109+
if (probability(SLOW_PATH_PROBABILITY, Continuation.isSupported() && hub == DynamicHub.fromClass(StoredContinuation.class))) {
110+
finishFormatStoredContinuation(obj);
111+
}
112+
emitMemoryBarrierIf(emitMemoryBarrier);
113+
return obj;
107114
}
108115

109116
private static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, boolean unaligned) {
110117
return ObjectHeaderImpl.encodeAsObjectHeader(hub, rememberedSet, unaligned);
111118
}
112119

120+
@Snippet
121+
public Object allocateStoredContinuationInstance(@Snippet.NonNullParameter DynamicHub hub, int size, @ConstantParameter AllocationProfilingData profilingData) {
122+
int baseOffset = StoredContinuationImpl.PAYLOAD_OFFSET;
123+
Object result = allocateArrayImpl(encodeAsTLABObjectHeader(hub), WordFactory.nullPointer(), size, baseOffset, 0,
124+
FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED, baseOffset, false, false, false, false, profilingData);
125+
finishFormatStoredContinuation(result);
126+
emitMemoryBarrierIf(true);
127+
return PiNode.piCastToSnippetReplaceeStamp(result);
128+
}
129+
130+
private static void finishFormatStoredContinuation(Object obj) {
131+
StoredContinuationImpl.initializeNewlyAllocated(obj);
132+
}
133+
113134
@Override
114135
public void initializeObjectHeader(Word memory, Word objectHeader, Word prototypeMarkWord, boolean isArray) {
115136
Heap.getHeap().getObjectHeader().initializeHeaderOfNewObject(memory, objectHeader);
@@ -158,12 +179,14 @@ protected SubstrateForeignCallDescriptor getSlowNewArrayStub() {
158179
public static class Templates extends SubstrateAllocationSnippets.Templates {
159180
private final SnippetInfo formatObject;
160181
private final SnippetInfo formatArray;
182+
private final SnippetInfo allocateStoredContinuationInstance;
161183

162184
Templates(SubstrateAllocationSnippets receiver, OptionValues options, SnippetCounter.Group.Factory groupFactory, Providers providers) {
163185
super(receiver, options, groupFactory, providers);
164186

165187
formatObject = snippet(GenScavengeAllocationSnippets.class, "formatObjectSnippet", null, receiver);
166188
formatArray = snippet(GenScavengeAllocationSnippets.class, "formatArraySnippet", null, receiver);
189+
allocateStoredContinuationInstance = snippet(GenScavengeAllocationSnippets.class, "allocateStoredContinuationInstance", null, receiver, ALLOCATION_LOCATIONS);
167190
}
168191

169192
@Override
@@ -175,6 +198,9 @@ public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>
175198

176199
FormatArrayLowering formatArrayLowering = new FormatArrayLowering();
177200
lowerings.put(FormatArrayNode.class, formatArrayLowering);
201+
202+
NewStoredContinuationLowering newStoredContinuationLowering = new NewStoredContinuationLowering();
203+
lowerings.put(NewStoredContinuationNode.class, newStoredContinuationLowering);
178204
}
179205

180206
private class FormatObjectLowering implements NodeLoweringProvider<FormatObjectNode> {
@@ -187,7 +213,6 @@ public void lower(FormatObjectNode node, LoweringTool tool) {
187213
Arguments args = new Arguments(formatObject, graph.getGuardsStage(), tool.getLoweringStage());
188214
args.add("memory", node.getMemory());
189215
args.add("hub", node.getHub());
190-
args.add("size", node.getSize());
191216
args.add("rememberedSet", node.getRememberedSet());
192217
args.add("fillContents", node.getFillContents());
193218
args.add("emitMemoryBarrier", node.getEmitMemoryBarrier());
@@ -218,5 +243,28 @@ public void lower(FormatArrayNode node, LoweringTool tool) {
218243
template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
219244
}
220245
}
246+
247+
private class NewStoredContinuationLowering implements NodeLoweringProvider<NewStoredContinuationNode> {
248+
@Override
249+
public void lower(NewStoredContinuationNode node, LoweringTool tool) {
250+
StructuredGraph graph = node.graph();
251+
252+
if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
253+
return;
254+
}
255+
256+
DynamicHub hub = ((SharedType) tool.getMetaAccess().lookupJavaType(StoredContinuation.class)).getHub();
257+
assert hub.isStoredContinuationClass();
258+
259+
ConstantNode hubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(hub), providers.getMetaAccess(), graph);
260+
261+
Arguments args = new Arguments(allocateStoredContinuationInstance, graph.getGuardsStage(), tool.getLoweringStage());
262+
args.add("hub", hubConstant);
263+
args.add("size", node.getSize());
264+
args.addConst("profilingData", getProfilingData(node, null));
265+
266+
template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
267+
}
268+
}
221269
}
222270
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatObjectNode.java‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,24 +35,21 @@
3535
import org.graalvm.compiler.nodes.spi.Lowerable;
3636
import org.graalvm.compiler.replacements.AllocationSnippets.FillContent;
3737
import org.graalvm.word.Pointer;
38-
import org.graalvm.word.UnsignedWord;
3938

4039
@NodeInfo(cycles = CYCLES_64, size = SIZE_64)
4140
public class FormatObjectNode extends FixedWithNextNode implements Lowerable {
4241
public static final NodeClass<FormatObjectNode> TYPE = NodeClass.create(FormatObjectNode.class);
4342

4443
@Input protected ValueNode memory;
4544
@Input protected ValueNode hub;
46-
@Input protected ValueNode size;
4745
@Input protected ValueNode rememberedSet;
4846
@Input protected ValueNode fillContents;
4947
@Input protected ValueNode emitMemoryBarrier;
5048

51-
public FormatObjectNode(ValueNode memory, ValueNode hub, ValueNode size, ValueNode rememberedSet, ValueNode fillContents, ValueNode emitMemoryBarrier) {
49+
public FormatObjectNode(ValueNode memory, ValueNode hub, ValueNode rememberedSet, ValueNode fillContents, ValueNode emitMemoryBarrier) {
5250
super(TYPE, StampFactory.objectNonNull());
5351
this.memory = memory;
5452
this.hub = hub;
55-
this.size = size;
5653
this.rememberedSet = rememberedSet;
5754
this.fillContents = fillContents;
5855
this.emitMemoryBarrier = emitMemoryBarrier;
@@ -66,10 +63,6 @@ public ValueNode getHub() {
6663
return hub;
6764
}
6865

69-
public ValueNode getSize() {
70-
return size;
71-
}
72-
7366
public ValueNode getRememberedSet() {
7467
return rememberedSet;
7568
}
@@ -83,5 +76,5 @@ public ValueNode getEmitMemoryBarrier() {
8376
}
8477

8578
@NodeIntrinsic
86-
public static native Object formatObject(Pointer memory, Class<?> hub, UnsignedWord size, boolean rememberedSet, FillContent fillContents, boolean emitMemoryBarrier);
79+
public static native Object formatObject(Pointer memory, Class<?> hub, boolean rememberedSet, FillContent fillContents, boolean emitMemoryBarrier);
8780
}

‎substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewStoredContinuationNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,5 @@ public ValueNode getSize() {
4949
}
5050

5151
@NodeIntrinsic
52-
public static native StoredContinuation allocate(long size);
52+
public static native StoredContinuation allocate(int size);
5353
}

0 commit comments

Comments
 (0)