[GR-57317] Update and clean up patches, part #1

PullRequest: graalpython/3449

Author: msimacek

docs/user/Native-Extensions.md

@@ -7,7 +7,7 @@ permalink: /reference-manual/python/Native-Extensions/
 
 # Native Extensions Support
 
-CPython provides a [native extensions API](https://docs.python.org/3/c-api/index.html){:target="_blank"} for writing Python extensions in C/C++.
+CPython provides a [native extensions API](https://docs.python.org/3/c-api/index.html) for writing Python extensions in C/C++.
 GraalPy provides experimental support for this API, which allows many packages like NumPy and PyTorch to work well for many use cases.
 The support extends only to the API, not the binary interface (ABI), so extensions built for CPython are not binary compatible with GraalPy.
 Packages that use the native API must be built and installed with GraalPy, and the prebuilt wheels for CPython from pypi.org cannot be used.
@@ -18,7 +18,7 @@ Please do not update `pip` or use alternative tools such as `uv`.
 ## Embedding limitations
 
 Python native extensions run by default as native binaries, with full access to the underlying system.
-Native code is not sandboxed and can circumvent any protections Truffle or the JVM may provide, up to and including aborting the entire process.
+Native code is entirely unrestricted and can circumvent any security protections Truffle or the JVM may provide.
 Native data structures are not subject to the Java GC and the combination of them with Java data structures may lead to memory leaks.
 Native libraries generally cannot be loaded multiple times into the same process, and they may contain global state that cannot be safely reset.
 Thus, it is not possible to create multiple GraalPy contexts that access native modules within the same JVM.

graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py

@@ -59,24 +59,50 @@ def check_autopatched(source, expected):
 
     def test_replace_field_access():
         check_autopatched(
-            """
-            PyObject* type = obj->ob_type;
-            PyObject* type = ((PyObject*)obj)->ob_type;
-            const char* name = obj->ob_type->tp_name;
-            foo = objs[0]->ob_type;
-            return obj->ob_type;
-            return (PyObject*)obj->ob_type;
-            return ((PyObject*)obj)->ob_type;
-            (PyObject*)function_call(a, b, c(0))->ob_type->ob_base;
-            """,
-            """
-            PyObject* type = Py_TYPE(obj);
-            PyObject* type = Py_TYPE(((PyObject*)obj));
-            const char* name = Py_TYPE(obj)->tp_name;
-            foo = Py_TYPE(objs[0]);
-            return Py_TYPE(obj);
-            return (PyObject*)Py_TYPE(obj);
-            return Py_TYPE(((PyObject*)obj));
-            (PyObject*)Py_TYPE(function_call(a, b, c(0)))->ob_base;
-            """
+            'PyObject* type = obj->ob_type;',
+            'PyObject* type = Py_TYPE(obj);',
+        )
+        check_autopatched(
+            'PyObject* type = ((PyObject*)obj)->ob_type;',
+            'PyObject* type = Py_TYPE(((PyObject*)obj));',
+        )
+        check_autopatched(
+            'const char* name = obj->ob_type->tp_name;',
+            'const char* name = Py_TYPE(obj)->tp_name;',
+        )
+        check_autopatched(
+            'foo = objs[0]->ob_type;',
+            'foo = Py_TYPE(objs[0]);',
+        )
+        check_autopatched(
+            'return obj->ob_type;',
+            'return Py_TYPE(obj);',
+        )
+        check_autopatched(
+            'return (PyObject*)obj->ob_type;',
+            'return (PyObject*)Py_TYPE(obj);',
+        )
+        check_autopatched(
+            'return ((PyObject*)obj)->ob_type;',
+            'return Py_TYPE(((PyObject*)obj));',
+        )
+        check_autopatched(
+            '(PyObject*)function_call(a, b, c(0))->ob_type->ob_base;',
+            '(PyObject*)Py_TYPE(function_call(a, b, c(0)))->ob_base;',
+        )
+        check_autopatched(
+            '''
+            #if SOME_MACRO
+            obj->ob_type->tp_free(self);
+            #else
+            obj->ob_type->tp_free(self);
+            #endif
+            ''',
+            '''
+            #if SOME_MACRO
+            Py_TYPE(obj)->tp_free(self);
+            #else
+            Py_TYPE(obj)->tp_free(self);
+            #endif
+            ''',
         )

graalpython/lib-graalpython/_sysconfig.py

@@ -102,18 +102,18 @@ def _get_posix_vars():
     g['CFLAGS'] = ' '.join(cflags_default + [gnu_source])
     g['LDFLAGS'] = ""
     g['CCSHARED'] = fpic
-    g['LDSHARED_LINUX'] = "%s -shared %s" % (get_toolchain('CC'), fpic)
     if darwin_native:
-        g['LDSHARED'] = get_toolchain('CC') + " -bundle -undefined dynamic_lookup"
         g['LDFLAGS'] = "-bundle -undefined dynamic_lookup"
+        ldshared_common = g['LDFLAGS']
         g['LIBPYTHON'] = ''
     elif win32_native:
         g['LDFLAGS'] = f"-L{__graalpython__.capi_home.replace(os.path.sep, '/')}"
-        g['LDSHARED_WINDOWS'] = f"{g['LDSHARED_LINUX']} {g['LDFLAGS']}"
-        g['LDSHARED'] = g['LDSHARED_WINDOWS']
+        ldshared_common = f"-shared {fpic} {g['LDFLAGS']}"
     else:
-        g['LDSHARED'] = g['LDSHARED_LINUX']
+        ldshared_common = f"-shared {fpic}"
         g['LIBPYTHON'] = ''
+    g['LDSHARED'] = f"{g['CC']} {ldshared_common}"
+    g['LDCXXSHARED'] = f"{g['CXX']} {ldshared_common}"
     g['SOABI'] = so_abi
     g['EXT_SUFFIX'] = "." + so_abi + so_ext
     g['SHLIB_SUFFIX'] = so_ext

graalpython/lib-graalpython/modules/autopatch_capi.py

@@ -66,7 +66,7 @@ def replace_field_access(contents, match, replacement, assignment):
     level = 0
 
     def consume_whitespace_backwards(idx):
-        while idx > 0 and contents[idx].isspace():
+        while idx >= 0 and contents[idx].isspace():
             idx -= 1
         return idx
 
@@ -85,7 +85,7 @@ def consume_whitespace_forward(idx):
     def consume_pairwise_backwards(idx, l, r):
         level = 1
         idx -= 1
-        while level and idx:
+        while level and idx >= 0:
             c = contents[idx]
             if c == l:
                 level -= 1
@@ -95,31 +95,35 @@ def consume_pairwise_backwards(idx, l, r):
         return idx
 
     def consume_identifier_backwards(idx):
-        while (contents[idx].isidentifier() or contents[idx].isdigit()) and idx:
+        while idx >= 0 and (contents[idx].isidentifier() or contents[idx].isdigit()):
             idx -= 1
         return idx
 
-    first = True
-    while idx:
+    allowed_tokens = {'()', '[]', '.', 'id'}
+    while idx >= 0:
         c = contents[idx]
-        if c == ')' and first:
+        if '()' in allowed_tokens and c == ')':
             idx = consume_pairwise_backwards(idx, '(', ')')
-        elif c == ']':
+            allowed_tokens = {'[]', 'id'}
+        elif '[]' in allowed_tokens and c == ']':
             idx = consume_pairwise_backwards(idx, '[', ']')
-        elif c.isidentifier() or c.isdigit():
+            allowed_tokens = {'[]', 'id'}
+        elif 'id' in allowed_tokens and c.isidentifier() or c.isdigit():
             id_start = consume_identifier_backwards(idx)
             if contents[id_start + 1: idx + 1] == 'return':
                 idx += 1
                 break
             idx = id_start
-        elif c == '.':
+            allowed_tokens = {'.'}
+        elif '.' in allowed_tokens and c == '.':
             idx -= 1
-        elif c == '>' and idx > 1 and contents[idx - 1] == '-':
+            allowed_tokens = {'[]', 'id'}
+        elif '.' in allowed_tokens and c == '>' and idx > 1 and contents[idx - 1] == '-':
             idx -= 2
+            allowed_tokens = {'[]', 'id'}
         else:
             idx += 1
             break
-        first = False
         idx = consume_whitespace_backwards(idx)
 
     receiver_start = consume_whitespace_forward(idx)

graalpython/lib-graalpython/patches/Cython/metadata.toml

@@ -1,5 +1,5 @@
 [[rules]]
-version = '== 3.0.10'
+version = '>= 3.0.10, < 3.0.12'
 patch = 'Cython-3.0.10.patch'
 license = 'Apache-2.0'
 

graalpython/lib-graalpython/patches/atomicwrites/atomicwrites.patch

@@ -1,4 +1,7 @@
 [[rules]]
+# With pyo3 changes upstreamed, this patch is not needed in recent versions
+# This old version is still used in ray's tests
+install-priority = 0
 version = '== 4.0.1'
 patch = 'bcrypt-4.0.1.patch'
 license = 'Apache-2.0'