Fix inlining depth check

Author: Christian Wimmer

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java

@@ -133,13 +133,20 @@ private boolean shouldInlineInvoke0(GraphBuilderContext b, SVMHost hostVM, Accum
             return true;
         }
 
-        boolean inMethodHandleIntrinsification = policyScope != null ? policyScope.accumulativeCounters.inMethodHandleIntrinsification : false;
+        boolean inMethodHandleIntrinsification = policyScope != null && policyScope.accumulativeCounters.inMethodHandleIntrinsification;
         int allowedInlinings = inMethodHandleIntrinsification ? optionMethodHandleAllowedInlinings : optionAllowedInlinings;
         if (policyScope != null && policyScope.accumulativeCounters.totalInlinedMethods >= allowedInlinings) {
             return false;
         }
         int allowedDepth = inMethodHandleIntrinsification ? optionMethodHandleAllowedDepth : optionAllowedDepth;
-        if (b.getDepth() >= allowedDepth) {
+        /*
+         * Note that we do not use the inlining depth from the GraphBuilderContext: If we are in a
+         * regular inlining scope, but nested into a deep method handle intrinsification, then the
+         * total inlining depth is high but our local depth for the scope can still be low enough to
+         * do inlining.
+         */
+        int actualDepth = policyScope == null ? 0 : policyScope.inliningDepth;
+        if (actualDepth >= allowedDepth) {
             return false;
         }
         if (!inMethodHandleIntrinsification && b.recursiveInliningDepth(method) > 0) {
@@ -284,8 +291,11 @@ public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineS
              * This assumes that the regular limits are strict enough to prevent excessive inlining
              * triggered by method handles. We could also use alternative fixed values or the option
              * defaults instead of any set option values.
+             *
+             * We start again with an inlining depth of 1, i.e., we behave as if that method is the
+             * inlining root.
              */
-            depth = outer.inliningDepth + 1;
+            depth = 1;
             accumulativeCounters = new AccumulativeCounters(optionAllowedNodes, optionAllowedInvokes, false);
 
         } else {