[GR-51002] Improve intrinsification of method handles.

PullRequest: graal/16410

Author: Christian Wimmer

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java

@@ -344,7 +344,7 @@ public String getNodeClassName() {
     }
 
     protected class PENonAppendGraphBuilderContext extends CoreProvidersDelegate implements GraphBuilderContext {
-        protected final PEMethodScope methodScope;
+        public final PEMethodScope methodScope;
         protected final Invoke invoke;
 
         @Override

substratevm/CHANGELOG.md

@@ -18,6 +18,7 @@ This changelog summarizes major changes to GraalVM Native Image.
 * (GR-30433) Disallow the deprecated environment variable USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM=false.
 * (GR-49655) Experimental support for parts of the [Foreign Function & Memory API](https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/ForeignInterface.md) (part of "Project Panama", [JEP 454](https://openjdk.org/jeps/454)) on AMD64. Must be enabled with `-H:+ForeignAPISupport` (requiring `-H:+UnlockExperimentalVMOptions`).
 * (GR-46407) Correctly rethrow build-time linkage errors at run-time for registered reflection queries.
+* (GR-51002) Improve intrinsification of method handles. This especially improves the performance of `equals` and `hashCode` methods for records, which use method handles that are now intrinsified.
 
 ## GraalVM for JDK 21 (Internal Version 23.1.0)
 * (GR-35746) Lower the default aligned chunk size from 1 MB to 512 KB for the serial and epsilon GCs, reducing memory usage and image size in many cases.

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ObjectScanner.java

@@ -145,7 +145,7 @@ protected void scanEmbeddedRoot(JavaConstant root, Object position) {
         try {
             scanningObserver.forEmbeddedRoot(root, reason);
             scanConstant(root, reason);
-        } catch (UnsupportedFeatureException ex) {
+        } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) {
             bb.getUnsupportedFeatures().addMessage(reason.toString(), reason.getMethod(), ex.getMessage(), null, ex);
         }
     }
@@ -205,7 +205,7 @@ protected void scanField(AnalysisField field, JavaConstant receiver, ScanReason
                 scanningObserver.forPrimitiveFieldValue(receiver, field, fieldValue, reason);
             }
 
-        } catch (UnsupportedFeatureException ex) {
+        } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) {
             unsupportedFeatureDuringFieldScan(bb, field, receiver, ex, reason);
         }
     }
@@ -268,7 +268,7 @@ protected final void scanArray(JavaConstant array, ScanReason prevReason) {
                     try {
                         JavaConstant element = bb.getUniverse().getSnippetReflection().forObject(bb.getUniverse().replaceObject(e));
                         scanArrayElement(array, arrayType, reason, idx, element);
-                    } catch (UnsupportedFeatureException ex) { /* Object replacement can throw. */
+                    } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) {
                         unsupportedFeatureDuringConstantScan(bb, bb.getUniverse().getSnippetReflection().forObject(e), ex, reason);
                     }
                 }
@@ -313,7 +313,7 @@ public void scanConstant(JavaConstant value, ScanReason reason) {
      * Use the constant hashCode as a key for the unsupported feature to register only one error
      * message if the constant is reachable from multiple places.
      */
-    public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant constant, UnsupportedFeatureException e, ScanReason reason) {
+    public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant constant, Throwable e, ScanReason reason) {
         unsupportedFeature(bb, String.valueOf(receiverHashCode(constant)), e.getMessage(), reason);
     }
 
@@ -322,11 +322,11 @@ public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant
      * only one error message if the value is reachable from multiple places. For example both the
      * heap scanning and the heap verification would scan a field that contains an illegal value.
      */
-    public static void unsupportedFeatureDuringFieldScan(BigBang bb, AnalysisField field, JavaConstant receiver, UnsupportedFeatureException e, ScanReason reason) {
+    public static void unsupportedFeatureDuringFieldScan(BigBang bb, AnalysisField field, JavaConstant receiver, Throwable e, ScanReason reason) {
         unsupportedFeature(bb, (receiver != null ? receiverHashCode(receiver) + "_" : "") + field.format("%H.%n"), e.getMessage(), reason);
     }
 
-    public static void unsupportedFeatureDuringFieldFolding(BigBang bb, AnalysisField field, JavaConstant receiver, UnsupportedFeatureException e, AnalysisMethod parsedMethod, int bci) {
+    public static void unsupportedFeatureDuringFieldFolding(BigBang bb, AnalysisField field, JavaConstant receiver, Throwable e, AnalysisMethod parsedMethod, int bci) {
         ScanReason reason = new FieldConstantFold(field, parsedMethod, bci, receiver, new MethodParsing(parsedMethod));
         unsupportedFeature(bb, (receiver != null ? receiverHashCode(receiver) + "_" : "") + field.format("%H.%n"), e.getMessage(), reason);
     }
@@ -448,7 +448,7 @@ private void doScan(WorklistEntry entry) {
                 /* Scan the array elements. */
                 scanArray(entry.constant, entry.reason);
             }
-        } catch (UnsupportedFeatureException ex) {
+        } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) {
             unsupportedFeatureDuringConstantScan(bb, entry.constant, ex, entry.reason);
         }
     }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java

@@ -595,6 +595,9 @@ public void rescanField(Object receiver, Field reflectionField, ScanReason reaso
             if (type.isReachable()) {
                 AnalysisField field = metaAccess.lookupJavaField(reflectionField);
                 assert !field.isStatic() : field;
+                if (!field.isReachable()) {
+                    return;
+                }
                 JavaConstant receiverConstant = asConstant(receiver);
                 Optional<JavaConstant> replaced = maybeReplace(receiverConstant, reason);
                 if (replaced.isPresent()) {

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java

@@ -38,6 +38,7 @@
 import com.oracle.graal.pointsto.meta.AnalysisField;
 import com.oracle.graal.pointsto.meta.AnalysisMethod;
 import com.oracle.graal.pointsto.meta.HostedProviders;
+import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy.AbstractPolicyScope;
 import com.oracle.graal.pointsto.util.AnalysisError;
 import com.oracle.graal.pointsto.util.GraalAccess;
 import com.oracle.svm.util.ReflectionUtil;
@@ -60,6 +61,7 @@
 import jdk.graal.compiler.nodes.ValueNode;
 import jdk.graal.compiler.nodes.calc.IsNullNode;
 import jdk.graal.compiler.nodes.extended.UnsafeAccessNode;
+import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin;
 import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin;
@@ -78,7 +80,7 @@ public class InlineBeforeAnalysisGraphDecoder extends PEGraphDecoder {
 
     public class InlineBeforeAnalysisMethodScope extends PEMethodScope {
 
-        public final InlineBeforeAnalysisPolicy.AbstractPolicyScope policyScope;
+        public final AbstractPolicyScope policyScope;
 
         private boolean inliningAborted;
 
@@ -103,7 +105,7 @@ public class InlineBeforeAnalysisMethodScope extends PEMethodScope {
                 for (int i = 0; i < arguments.length; i++) {
                     constArgsWithReceiver[i] = arguments[i].isConstant();
                 }
-                policyScope = policy.openCalleeScope(cast(caller).policyScope, bb.getMetaAccess(), method, constArgsWithReceiver, invokeData.intrinsifiedMethodHandle);
+                policyScope = policy.openCalleeScope(cast(caller).policyScope, method);
                 if (graph.getDebug().isLogEnabled()) {
                     graph.getDebug().logv("  ".repeat(inliningDepth) + "openCalleeScope for " + method.format("%H.%n(%p)") + ": " + policyScope);
                 }
@@ -121,6 +123,27 @@ static void recordInlined(InlineBeforeAnalysisMethodScope callerScope, InlineBef
         }
     }
 
+    static final class InlineBeforeAnalysisInlineInvokePlugin implements InlineInvokePlugin {
+
+        private final InlineBeforeAnalysisPolicy policy;
+
+        InlineBeforeAnalysisInlineInvokePlugin(InlineBeforeAnalysisPolicy policy) {
+            this.policy = policy;
+        }
+
+        @Override
+        public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod m, ValueNode[] args) {
+            AnalysisMethod method = (AnalysisMethod) m;
+
+            AbstractPolicyScope policyScope = cast(((PENonAppendGraphBuilderContext) b).methodScope).policyScope;
+            if (policy.shouldInlineInvoke(b, policyScope, method, args)) {
+                return policy.createInvokeInfo(method);
+            } else {
+                return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION;
+            }
+        }
+    }
+
     private Field dmhStaticAccessorOffsetField;
     private Field dmhStaticAccessorBaseField;
     private AnalysisField dmhStaticAccessorOffsetAnalysisField;
@@ -301,8 +324,19 @@ private void ensureDMHStaticAccessorFieldsInitialized() {
     }
 
     @Override
-    protected void handleNonInlinedInvoke(MethodScope methodScope, LoopScope loopScope, InvokeData invokeData) {
+    protected void handleNonInlinedInvoke(MethodScope ms, LoopScope loopScope, InvokeData invokeData) {
+        InlineBeforeAnalysisMethodScope methodScope = cast(ms);
         maybeAbortInlining(methodScope, loopScope, invokeData.invoke.asNode());
+
+        if (!methodScope.inliningAborted && methodScope.isInlinedMethod()) {
+            if (graph.getDebug().isLogEnabled()) {
+                graph.getDebug().logv("  ".repeat(methodScope.inliningDepth) + "  nonInlinedInvoke " + invokeData.callTarget.targetMethod() + ": " + methodScope.policyScope);
+            }
+            if (!methodScope.policyScope.processNonInlinedInvoke(providers, invokeData.callTarget)) {
+                abortInlining(methodScope);
+            }
+        }
+
         super.handleNonInlinedInvoke(methodScope, loopScope, invokeData);
     }
 
@@ -494,7 +528,7 @@ private void killControlFlowNodes(PEMethodScope inlineScope, FixedNode start) {
      * at the cost of this ugly cast.
      */
     @SuppressWarnings("unchecked")
-    protected InlineBeforeAnalysisMethodScope cast(MethodScope methodScope) {
+    protected static InlineBeforeAnalysisMethodScope cast(MethodScope methodScope) {
         return (InlineBeforeAnalysisMethodScope) methodScope;
     }
 

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java

@@ -30,11 +30,13 @@
 
 import jdk.graal.compiler.graph.Node;
 import jdk.graal.compiler.graph.NodeSourcePosition;
+import jdk.graal.compiler.nodes.CallTargetNode;
 import jdk.graal.compiler.nodes.FixedWithNextNode;
 import jdk.graal.compiler.nodes.ValueNode;
 import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo;
 import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin;
+import jdk.graal.compiler.nodes.spi.CoreProviders;
 
 /**
  * Provides the policy which methods are inlined by {@link InlineBeforeAnalysis}. If
@@ -77,6 +79,8 @@ protected AbstractPolicyScope(int inliningDepth) {
          * decision on the current list of usages. The list of usages is often but not always empty.
          */
         public abstract boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, Node node);
+
+        public abstract boolean processNonInlinedInvoke(CoreProviders providers, CallTargetNode node);
     }
 
     protected final NodePlugin[] nodePlugins;
@@ -85,7 +89,7 @@ protected InlineBeforeAnalysisPolicy(NodePlugin[] nodePlugins) {
         this.nodePlugins = nodePlugins;
     }
 
-    protected abstract boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args);
+    protected abstract boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args);
 
     protected abstract InlineInfo createInvokeInfo(AnalysisMethod method);
 
@@ -97,8 +101,7 @@ protected InlineBeforeAnalysisPolicy(NodePlugin[] nodePlugins) {
 
     protected abstract AbstractPolicyScope createRootScope();
 
-    protected abstract AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess,
-                    AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle);
+    protected abstract AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method);
 
     /** @see InlineBeforeAnalysisGraphDecoder#shouldOmitIntermediateMethodInStates */
     protected boolean shouldOmitIntermediateMethodInState(AnalysisMethod method) {
@@ -108,7 +111,7 @@ protected boolean shouldOmitIntermediateMethodInState(AnalysisMethod method) {
     public static final InlineBeforeAnalysisPolicy NO_INLINING = new InlineBeforeAnalysisPolicy(new NodePlugin[0]) {
 
         @Override
-        protected boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args) {
+        protected boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args) {
             return false;
         }
 
@@ -143,8 +146,7 @@ protected AbstractPolicyScope createRootScope() {
         }
 
         @Override
-        protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess,
-                        AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) {
+        protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method) {
             throw AnalysisError.shouldNotReachHere("NO_INLINING policy should not try to inline");
         }
     };

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java

@@ -27,6 +27,7 @@
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.VarHandle;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.HashMap;
 import java.util.Map;
@@ -99,6 +100,70 @@ public class VarHandleFeature implements InternalFeature {
     private final ConcurrentMap<Object, Boolean> processedVarHandles = new ConcurrentHashMap<>();
     private Consumer<Field> markAsUnsafeAccessed;
 
+    @Override
+    public void duringSetup(DuringSetupAccess access) {
+        /*
+         * Initialize fields of VarHandle instances that are @Stable eagerly, so that during method
+         * handle intrinsification loads of those fields and array elements can be constant-folded.
+         * 
+         * Note that we do this on purpose here in an object replacer, and not in an object
+         * reachability handler: Intrinsification happens as part of method inlining before
+         * analysis, i.e., before the static analysis, i.e., before the VarHandle object itself is
+         * marked as reachable. The goal of intrinsification is to actually avoid making the
+         * VarHandle object itself reachable.
+         */
+        access.registerObjectReplacer(VarHandleFeature::eagerlyInitializeVarHandle);
+    }
+
+    private static Object eagerlyInitializeVarHandle(Object obj) {
+        if (obj instanceof VarHandle varHandle) {
+            eagerlyInitializeVarHandle(varHandle);
+        }
+        return obj;
+    }
+
+    private static final Field varHandleVFormField = ReflectionUtil.lookupField(VarHandle.class, "vform");
+    private static final Method varFormInitMethod = ReflectionUtil.lookupMethod(ReflectionUtil.lookupClass(false, "java.lang.invoke.VarForm"), "getMethodType_V", int.class);
+    private static final Method varHandleGetMethodHandleMethod = ReflectionUtil.lookupMethod(VarHandle.class, "getMethodHandle", int.class);
+
+    public static void eagerlyInitializeVarHandle(VarHandle varHandle) {
+        try {
+            /*
+             * The field VarHandle.vform.methodType_V_table is a @Stable field but initialized
+             * lazily on first access. Therefore, constant folding can happen only after
+             * initialization has happened. We force initialization by invoking the method
+             * VarHandle.vform.getMethodType_V(0).
+             */
+            Object varForm = varHandleVFormField.get(varHandle);
+            varFormInitMethod.invoke(varForm, 0);
+
+            /*
+             * The AccessMode used for the access that we are going to intrinsify is hidden in a
+             * AccessDescriptor object that is also passed in as a parameter to the intrinsified
+             * method. Initializing all AccessMode enum values is easier than trying to extract the
+             * actual AccessMode.
+             */
+            for (VarHandle.AccessMode accessMode : VarHandle.AccessMode.values()) {
+                /*
+                 * Force initialization of the @Stable field VarHandle.vform.memberName_table.
+                 */
+                boolean isAccessModeSupported = varHandle.isAccessModeSupported(accessMode);
+                /*
+                 * Force initialization of the @Stable field
+                 * VarHandle.typesAndInvokers.methodType_table.
+                 */
+                varHandle.accessModeType(accessMode);
+
+                if (isAccessModeSupported) {
+                    /* Force initialization of the @Stable field VarHandle.methodHandleTable. */
+                    varHandleGetMethodHandleMethod.invoke(varHandle, accessMode.ordinal());
+                }
+            }
+        } catch (ReflectiveOperationException ex) {
+            throw VMError.shouldNotReachHere(ex);
+        }
+    }
+
     @Override
     public void afterRegistration(AfterRegistrationAccess access) {
         try {