[GR-48482] Python: backport multiple fixes.

PullRequest: graalpython/2952

Author: steve-s

CHANGELOG.md

@@ -4,17 +4,10 @@ This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 23.1.0
-* Oracle GraalPy standalones (also known as GraalPy Enterprise) are now available under GFTC. The community builds published on Github have been renamed to `graalpy-community-<version>-<os>-<arch>.tar.gz`.
+* Oracle GraalPy distributions (also known as GraalPy Enterprise) are now available under the [GFTC license](https://www.oracle.com/downloads/licenses/graal-free-license.html). The community builds published on Github have been renamed to `graalpy-community-<version>-<os>-<arch>.tar.gz`.
 * Add support for the sqlite3 module. This allows many packages like `coverage` or `Flask-SQLAlchemy` to work on top of this embedded database.
-* The GraalPy standalone tool was updated to build single-file executable Python binaries for Linux, Windows, and macOS.
-* The GraalPy standalone tool was updated to generate skeleton Maven projects the demonstrate polyglot embedding.
-* Support venv and pip installation of with pure Python wheels on Windows. This is the first preview of Windows support for this feature, and there are limitations, but pure Python packages like Pygal can be installed with `python -m pip --no-cache install pygal`.
-* Support compilation and execution of C extensions using the native MSVC toolchain on Windows. This feature allows building C extensions from source, installation via pip is not possible at this time.
-* We have [contributed](https://github.com/actions/setup-python/pull/694) support for GraalPy to GitHub's `setup-python` action. We hope it will be available in the default repository soon, until then you can try it from the fork to run your Python workloads on GraalPy.
-* Added support for the sqlite3 module. This allows many packages like `coverage` or `Flask-SQLAlchemy` to work on top of this embedded database.
-* The GraalPy standalone tool was updated. You can now build single-file executable Python binaries for Linux, Windows, and macOS as well as generate skeleton Maven projects that set up a polyglot embedding of Python packages into Java.
-* Support venv and pip installation of with pure Python wheels on Windows. This is the first preview of Windows support for this feature, and there are limitations, but pure Python packages like Pygal can be installed with `python -m pip --no-cache install pygal`.
-* Support compilation and execution of C extensions using the native MSVC toolchain on Windows. This feature allows building C extensions from source, installation via pip is not possible at this time.
+* Provide Windows distributions of GraalPy. This is the first preview of Windows support for this feature, and there are limitations, but pure Python packages like Pygal can be installed with `python -m pip --no-cache install pygal`.
+* The GraalPy standalone tool was updated. You can now build single-file executable Python binaries for Linux, Windows, and macOS. The tool can also generate a skeleton Maven project that sets up a polyglot embedding of Python packages into Java.
 
 ## Version 23.0.0
 * Update `numpy` and `pandas` versions, add support for `scipy` and `scikit_learn` with `ginstall`. This automatically applies some fixes that make it possible to use these new versions with GraalPy.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "911d7988b155cbf44fb0e4d545dd42083d8b3c90" }
+{ "overlay": "307e5a817ece2f16b7500db068a58c0538cf2101" }

graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py

@@ -210,7 +210,7 @@ def test_charptr_write_ptr(compiler, python_subprocess):
         assert result.stderr == b""
 
 
-@pytest.mark.xfail(__graalpython__.platform_id == "managed", reason="unaligned memcpy is not supported")
+@pytest.mark.xfail(__graalpython__.get_platform_id() == "managed", reason="unaligned memcpy is not supported")
 def test_charptr_correct_usage(compiler):
     mod = compiler.make_module("""
         #include <string.h>

graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py

@@ -32,8 +32,8 @@
 
 PY2 = sys.version_info[0] == 2
 GRAALPYTHON = sys.implementation.name == 'graalpy'
-GRAALPYTHON_NATIVE = GRAALPYTHON and __graalpython__.platform_id == 'native'
-DARWIN_NATIVE = sys.platform == 'darwin' and (not GRAALPYTHON or __graalpython__.platform_id == 'native')
+GRAALPYTHON_NATIVE = GRAALPYTHON and __graalpython__.get_platform_id() == 'native'
+DARWIN_NATIVE = sys.platform == 'darwin' and (not GRAALPYTHON or __graalpython__.get_platform_id() == 'native')
 
 HPY_ROOT = Path(__file__).parent.parent
 LOCK = FileLock(HPY_ROOT / ".hpy.lock")

graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybuildvalue.py

@@ -107,7 +107,7 @@ def test_formats(self):
             actual = mod.f(i)
             assert actual == expected, code
 
-    @pytest.mark.xfail(__graalpython__.platform_id == "managed", reason="GR-38126")
+    @pytest.mark.xfail(__graalpython__.get_platform_id() == "managed", reason="GR-38126")
     def test_bad_formats(self):
         test_cases = [
             ('return HPy_BuildValue(ctx, "(q)", 42);',

graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java

@@ -48,14 +48,13 @@
 
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.InternalResource;
-import com.oracle.truffle.api.InternalResource.Id;
 
 /**
  * This code needs to be kept in sync somehow with the suite.py code. In particular, the layouts
  * produced by the unpacking logic in {@link #unpackFiles} should produce the same layout as the
  * GRAALPYTHON_GRAALVM_SUPPORT distribution in the suite.py.
  */
-@Id("python-home")
+@InternalResource.Id(value = "python-home", componentId = "python", optional = true)
 public final class PythonResource implements InternalResource {
     private static final int PYTHON_MAJOR;
     private static final int PYTHON_MINOR;

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/advance/ExclusionsTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -41,11 +41,26 @@
 package com.oracle.graal.python.test.advance;
 
 import org.graalvm.polyglot.Context;
+import org.junit.Test;
 
 public class ExclusionsTest {
     public static void main(String[] args) {
         try (Context context = Context.create()) {
             context.eval("python", "print('Hello Python!');");
         }
     }
+
+    @Test
+    public void testDatetimeWithoutPlatformAccess() {
+        if (!"true".equals(System.getProperty("python.WithoutPlatformAccess"))) {
+            return;
+        }
+        var builder = Context.newBuilder().allowExperimentalOptions(true);
+        if (System.getenv("GRAAL_PYTHONHOME") != null) {
+            builder.option("python.PythonHome", System.getenv("GRAAL_PYTHONHOME"));
+        }
+        try (Context context = builder.build()) {
+            context.eval("python", "import datetime; datetime.datetime.strptime('2014 7 2 6 14 0 742 +0700', '%Y %m %d %H %M %S %f %z')");
+        }
+    }
 }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/advance/ResourcesTest.java

@@ -43,7 +43,9 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
+
 import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.io.IOAccess;
 import org.junit.Test;
 
 public class ResourcesTest {
@@ -59,4 +61,12 @@ public void testResourcesAsHome() {
             assertTrue(foundHome, !foundHome.contains("graalpython"));
         }
     }
+
+    @Test
+    public void testResourcesAlwaysAllowReading() {
+        try (Context context = Context.newBuilder("python").allowIO(IOAccess.NONE).option("python.PythonHome", "/path/that/does/not/exist").build()) {
+            String foundHome = context.eval("python", "import email; email.__spec__.origin").asString();
+            assertTrue(foundHome, foundHome.contains("python" + File.separator + "python-home"));
+        }
+    }
 }

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_gc.py

@@ -43,7 +43,7 @@
 from . import CPyExtTestCase, CPyExtFunction, CPyExtType
 __dir__ = __file__.rpartition("/")[0]
 
-GRAALPYTHON_NATIVE = sys.implementation.name == 'graalpy' and __graalpython__.platform_id == 'native'
+GRAALPYTHON_NATIVE = sys.implementation.name == 'graalpy' and __graalpython__.get_platform_id() == 'native'
 
 # typedef PyObject * (*unaryfunc)(PyObject *);
 # typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
@@ -92,7 +92,7 @@
             // printf("free\\n");
             free_cnt++;
             PyObject_Del(a);
-        }            
+        }
         static int dealloc_cnt = 0;
         void test_dealloc(PyObject *self) {
             // printf("dealloc\\n");
@@ -116,9 +116,9 @@
         tp_dealloc = "test_dealloc",
         tp_methods='{"getCounters", (PyCFunction)getCounters, METH_NOARGS | METH_STATIC, ""}, {"resetCounters", (PyCFunction)resetCounters, METH_NOARGS | METH_STATIC, ""}',
 )
-            
+
 class TestGC1():
-    
+
     def test_native_class(self):
         if GRAALPYTHON_NATIVE:
             gc.enable()
@@ -152,13 +152,13 @@ def test_native_class(self):
 #             Py_ssize_t r2 = Py_REFCNT(o);
 #             Py_DecRef(o);
 #             Py_ssize_t r3 = Py_REFCNT(o);
-#             return Py_BuildValue("(ii)", r2 - r1, r3 - r1); 
+#             return Py_BuildValue("(ii)", r2 - r1, r3 - r1);
 #         }''',
 #         resultspec="O",
 #         argspec='O',
 #         arguments=("PyObject* o", ),
 #         callfunction="wrap_simple",
-#     )    
+#     )
 #
 #
 #     test_create = CPyExtFunction(
@@ -171,10 +171,10 @@ def test_native_class(self):
 #             Py_ssize_t r2 = Py_REFCNT(o);
 #             Py_DecRef(o);
 #             Py_ssize_t r3 = Py_REFCNT(o);
-#             return Py_BuildValue("(ii)", r2 - r1, r3 - r1); 
+#             return Py_BuildValue("(ii)", r2 - r1, r3 - r1);
 #         }''',
 #         resultspec="O",
 #         argspec='O',
 #         arguments=("PyObject* o", ),
 #         callfunction="wrap_simple",
-#     )    
+#     )

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -39,6 +39,7 @@
 
 import itertools
 import sys
+import struct
 
 from . import CPyExtTestCase, CPyExtFunction, CPyExtType, unhandled_error_compare_with_message, unhandled_error_compare
 
@@ -445,6 +446,29 @@ def compile_module(self, name):
 
 
 class TestObject(object):
+    def test_memoryview_fromobject_multidim(self):
+        TestType = CPyExtType(
+            "TestMemoryViewMultidim",
+            """
+            PyObject* get_converted_view(PyObject* self, PyObject* obj) {
+                PyObject *mv = PyMemoryView_FromObject(obj);
+                if (!mv) {
+                    return NULL;
+                }
+
+                Py_buffer *view = PyMemoryView_GET_BUFFER(mv);
+                // int i = (int)PyNumber_AsSsize_t(idx, NULL);
+                return PyMemoryView_FromBuffer(view);
+            }
+            """,
+            tp_methods='{"get_converted_view", get_converted_view, METH_O, ""}',
+        )
+
+        obj = TestType()
+        t1_buf = struct.pack("d"*12, *[1.5*x for x in range(12)])
+        t1_mv = memoryview(t1_buf).cast('d', shape=[3,4])
+        assert t1_mv == obj.get_converted_view(t1_mv)
+
     def test_memoryview_acquire_release(self):
         TestType = CPyExtType(
             "TestMemoryViewBuffer1",