Optimize serial GC write barrier for non-array stores

Author: Christian Wimmer

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java

@@ -31,9 +31,11 @@
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.api.replacements.Snippet;
 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter;
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.nodes.BreakpointNode;
 import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
@@ -42,6 +44,7 @@
 import org.graalvm.compiler.nodes.gc.WriteBarrier;
 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.options.Option;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.util.Providers;
@@ -64,13 +67,18 @@
 import com.oracle.svm.core.util.Counter;
 import com.oracle.svm.core.util.CounterFeature;
 
+import jdk.vm.ci.meta.ResolvedJavaType;
+
 public class BarrierSnippets extends SubstrateTemplates implements Snippets {
     /** A LocationIdentity to distinguish card locations from other locations. */
     public static final LocationIdentity CARD_REMEMBERED_SET_LOCATION = NamedLocationIdentity.mutable("CardRememberedSet");
 
     public static class Options {
         @Option(help = "Instrument write barriers with counters")//
         public static final HostedOptionKey<Boolean> CountWriteBarriers = new HostedOptionKey<>(false);
+
+        @Option(help = "Verify write barriers")//
+        public static final HostedOptionKey<Boolean> VerifyWriteBarriers = new HostedOptionKey<>(true);
     }
 
     @Fold
@@ -90,23 +98,40 @@ public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>
     }
 
     @Snippet
-    public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean verifyOnly) {
+    public static void postWriteBarrierSnippet(@NonNullParameter Object object, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean verifyOnly) {
         counters().postWriteBarrier.inc();
 
         Object fixedObject = FixedValueAnchorNode.getObject(object);
         UnsignedWord objectHeader = ObjectHeaderImpl.readHeaderFromObject(fixedObject);
+
+        if (Options.VerifyWriteBarriers.getValue() && alwaysAlignedChunk) {
+            /*
+             * In addition to verifying that the object is in an aligned chunk, we also verify that
+             * it is not an array at all. Since most arrays are small and therefore in an aligned
+             * chunk, this greatly increases the verification coverage. To increase verification
+             * coverage, we also do the verification before checking if a barrier is needed at all.
+             */
+            if (ObjectHeaderImpl.isUnalignedHeader(objectHeader) || object.getClass().isArray()) {
+                BreakpointNode.breakpoint();
+            }
+        }
+
         boolean needsBarrier = RememberedSet.get().hasRememberedSet(objectHeader);
         if (BranchProbabilityNode.probability(BranchProbabilityNode.FREQUENT_PROBABILITY, !needsBarrier)) {
             return;
         }
-        boolean aligned = ObjectHeaderImpl.isAlignedHeader(objectHeader);
-        if (BranchProbabilityNode.probability(BranchProbabilityNode.LIKELY_PROBABILITY, aligned)) {
-            counters().postWriteBarrierAligned.inc();
-            RememberedSet.get().dirtyCardForAlignedObject(fixedObject, verifyOnly);
-            return;
+
+        if (!alwaysAlignedChunk) {
+            boolean unaligned = ObjectHeaderImpl.isUnalignedHeader(objectHeader);
+            if (BranchProbabilityNode.probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, unaligned)) {
+                counters().postWriteBarrierUnaligned.inc();
+                RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, verifyOnly);
+                return;
+            }
         }
-        counters().postWriteBarrierUnaligned.inc();
-        RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, verifyOnly);
+
+        counters().postWriteBarrierAligned.inc();
+        RememberedSet.get().dirtyCardForAlignedObject(fixedObject, verifyOnly);
     }
 
     private class PostWriteBarrierLowering implements NodeLoweringProvider<WriteBarrier> {
@@ -116,7 +141,20 @@ private class PostWriteBarrierLowering implements NodeLoweringProvider<WriteBarr
         public void lower(WriteBarrier barrier, LoweringTool tool) {
             Arguments args = new Arguments(postWriteBarrierSnippet, barrier.graph().getGuardsStage(), tool.getLoweringStage());
             OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress();
+
+            /*
+             * We know that instances (in contrast to arrays) are always in aligned chunks. There is
+             * no code anywhere that would allocate an instance into an unaligned chunk.
+             *
+             * Note that arrays can be assigned to values that have the type java.lang.Object, so
+             * that case is excluded. Arrays can also implement some interfaces, like Serializable.
+             * For simplicity, we exclude all interface types.
+             */
+            ResolvedJavaType baseType = StampTool.typeOrNull(address.getBase());
+            boolean alwaysAlignedChunk = baseType != null && !baseType.isArray() && !baseType.isJavaLangObject() && !baseType.isInterface();
+
             args.add("object", address.getBase());
+            args.addConst("alwaysAlignedChunk", alwaysAlignedChunk);
             args.addConst("verifyOnly", getVerifyOnly(barrier));
 
             template(barrier, args).instantiate(providers.getMetaAccess(), barrier, SnippetTemplate.DEFAULT_REPLACER, args);