GH-112215: Increase C recursion limit for non debug builds (GH-113397)

Author: markshannon

Include/cpython/pystate.h

@@ -223,9 +223,11 @@ struct _ts {
    // layout, optimization, and WASI runtime. Wasmtime can handle about 700
    // recursions, sometimes less. 500 is a more conservative limit.
 #  define Py_C_RECURSION_LIMIT 500
+#elif defined(__s390x__)
+#  define Py_C_RECURSION_LIMIT 1200
 #else
    // This value is duplicated in Lib/test/support/__init__.py
-#  define Py_C_RECURSION_LIMIT 1500
+#  define Py_C_RECURSION_LIMIT 8000
 #endif
 
 

Lib/test/support/__init__.py

@@ -2122,19 +2122,11 @@ def set_recursion_limit(limit):
         sys.setrecursionlimit(original_limit)
 
 def infinite_recursion(max_depth=None):
-    """Set a lower limit for tests that interact with infinite recursions
-    (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
-    debug windows builds, due to not enough functions being inlined the
-    stack size might not handle the default recursion limit (1000). See
-    bpo-11105 for details."""
     if max_depth is None:
-        if not python_is_optimized() or Py_DEBUG:
-            # Python built without compiler optimizations or in debug mode
-            # usually consumes more stack memory per function call.
-            # Unoptimized number based on what works under a WASI debug build.
-            max_depth = 50
-        else:
-            max_depth = 100
+        # Pick a number large enough to cause problems
+        # but not take too long for code that can handle
+        # very deep recursion.
+        max_depth = 20_000
     elif max_depth < 3:
         raise ValueError("max_depth must be at least 3, got {max_depth}")
     depth = get_recursion_depth()
@@ -2373,20 +2365,21 @@ def adjust_int_max_str_digits(max_digits):
     finally:
         sys.set_int_max_str_digits(current)
 
-#For recursion tests, easily exceeds default recursion limit
-EXCEEDS_RECURSION_LIMIT = 5000
 
 def _get_c_recursion_limit():
     try:
         import _testcapi
         return _testcapi.Py_C_RECURSION_LIMIT
     except (ImportError, AttributeError):
         # Originally taken from Include/cpython/pystate.h .
-        return 1500
+        return 8000
 
 # The default C recursion limit.
 Py_C_RECURSION_LIMIT = _get_c_recursion_limit()
 
+#For recursion tests, easily exceeds default recursion limit
+EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3
+
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
                                 'skipped on s390x')

Lib/test/test_functools.py

@@ -1862,6 +1862,20 @@ def orig(): ...
         self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()')
         self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()')
 
+    @support.skip_on_s390x
+    @unittest.skipIf(support.is_wasi, "WASI has limited C stack")
+    def test_lru_recursion(self):
+
+        @self.module.lru_cache
+        def fib(n):
+            if n <= 1:
+                return n
+            return fib(n-1) + fib(n-2)
+
+        if not support.Py_DEBUG:
+            with support.infinite_recursion():
+                fib(2500)
+
 
 @py_functools.lru_cache()
 def py_cached_func(x, y):

Lib/test/test_json/test_recursion.py

@@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self):
         for x in range(100000):
             l, d = [l], {'k':d}
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(5000):
                 self.dumps(l)
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(5000):
                 self.dumps(d)
 
     def test_endless_recursion(self):
@@ -99,7 +99,7 @@ def default(self, o):
                 return [o]
 
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(1000):
                 EndlessJSONEncoder(check_circular=False).encode(5j)
 
 

Lib/test/test_support.py

@@ -630,7 +630,7 @@ def recursive_function(depth):
             if depth:
                 recursive_function(depth - 1)
 
-        for max_depth in (5, 25, 250):
+        for max_depth in (5, 25, 250, 2500):
             with support.infinite_recursion(max_depth):
                 available = support.get_recursion_available()
 

Lib/test/test_xml_etree.py

@@ -2535,7 +2535,7 @@ def __eq__(self, o):
         e.extend([ET.Element('bar')])
         self.assertRaises(ValueError, e.remove, X('baz'))
 
-    @support.infinite_recursion(25)
+    @support.infinite_recursion()
     def test_recursive_repr(self):
         # Issue #25455
         e = ET.Element('foo')

Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst

@@ -0,0 +1,3 @@
+Increase the C recursion limit by a factor of 3 for non-debug builds, except
+for webassembly and s390 platforms which are unchanged. This mitigates some
+regressions in 3.12 with deep recursion mixing builtin (C) and Python code.