Merge commit 'dbc51fccb2a2b1857a0e3deeeaee0bd182630024' into release/graal-vm/1.0

Author: ansalond

CHANGELOG.md

@@ -3,6 +3,15 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 1.0.0 RC13
+
+* Support marshal.dumps and marshal.loads for code objects and some other built-in objects
+* Fix installation of NumPy in a venv
+* Initial support for module mmap
+* Support debugging with workspace files in the Chrome debugger
+* Support the PEP 553 breakpoint() message
+* Support running weak reference callbacks and signals on the main thread
+
 ## Version 1.0.0 RC12
 
 * Support the `__class__` variable in the class scope

ci.jsonnet

@@ -1,5 +1,5 @@
 {
-  overlay: "3cf78c3623442ad827eed58a1780784a6eb95676",
+  overlay: "d20cc2abdeb3cfb022e1a8035e40e350e5cfe5fc",
 
   // ======================================================================================================
   // 
@@ -88,6 +88,7 @@
 
   local darwinMixin = {
     capabilities +: ["darwin_sierra", "amd64"],
+    timelimit: TIME_LIMIT["1h"],
     packages +: {
       "pip:astroid": "==1.1.0",
       "pip:pylint": "==1.1.0",
@@ -127,7 +128,7 @@
 
   local labsjdk8Mixin = {
     downloads +: {
-      JAVA_HOME: utils.download("labsjdk", "8u172-jvmci-0.48"),
+      JAVA_HOME: utils.download("labsjdk", "8u192-jvmci-0.54"),
       EXTRA_JAVA_HOMES : { pathlist: [utils.download("oraclejdk", "11+20")] },
     },
     environment +: {

graalpython/benchmarks/src/micro/mmap-anonymous.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import mmap
+
+# create 64 MB anonymous mmap
+size = 64*1024
+mm = mmap.mmap(-1, size)
+data = b"Hello World"
+ndata = len(data)
+niter = size // ndata
+
+
+def fill():
+    # sequential access
+    mm.seek(0)
+    for i in range(niter):
+        mm.write(data)
+
+
+def measure(num):
+    for i in range(num):
+        fill()
+    print(mm[0:ndata])
+
+
+def __benchmark__(num=100):
+    measure(num)
+

graalpython/benchmarks/src/micro/mmap-file.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import mmap
+import tempfile
+
+# create temporary file
+data = b"Hello World"
+ndata = len(data)
+
+
+def fill(mm, size):
+    # sequential access
+    mm.seek(0)
+    for i in range(size // ndata):
+        mm.write(data)
+
+
+def measure(num):
+    tmp_path = tempfile.mkstemp()
+    with open(tmp_path[1], "wb") as f:
+        f.write(b'\x00' * (64 * 1024))
+
+    with open(tmp_path[1], "r+b") as f:
+        mm = mmap.mmap(f.fileno(), 0)
+        size = mm.size()
+        for i in range(num):
+            fill(mm, size)
+        print(mm[0:ndata])
+
+
+def __benchmark__(num=100):
+    measure(num)
+

graalpython/com.oracle.graal.python.cext/include/truffle.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2016, 2019, Oracle and/or its affiliates.
  *
  * All rights reserved.
  *
@@ -57,100 +57,6 @@ bool truffle_cannot_be_handle(void *nativeHandle);
 // wrapping functions
 void *truffle_decorate_function(void *function, void *wrapper);
 
-/*
- * All function below here are deprecated and will be removed in a future release.
- * Use the equivalent functions from <polyglot.h> instead.
- */
-
-void *truffle_import(const char *name);        // renamed to polyglot_import
-void *truffle_import_cached(const char *name); // no replacement, use polyglot_import
-
-void *truffle_address_to_function(void *address); // deprecated, does nothing
-
-void *truffle_get_arg(int i); // renamed to polyglot_get_arg
-
-// Predicates:
-bool truffle_is_executable(const void *object);     // renamed to polyglot_can_execute
-bool truffle_is_null(const void *object);           // renamed to polyglot_is_null
-bool truffle_has_size(const void *object);          // renamed to polyglot_has_array_elements
-bool truffle_is_boxed(const void *object);          // no replacement
-bool truffle_is_truffle_object(const void *object); // renamed to polyglot_is_value
-
-// Execute: deprecated, use typecast to function pointer instead
-void *truffle_execute(void *object, ...);
-int truffle_execute_i(void *object, ...);
-long truffle_execute_l(void *object, ...);
-char truffle_execute_c(void *object, ...);
-float truffle_execute_f(void *object, ...);
-double truffle_execute_d(void *object, ...);
-bool truffle_execute_b(void *object, ...);
-
-// Invoke:
-void *truffle_invoke(void *object, const char *name, ...);    // renamed to polyglot_invoke
-int truffle_invoke_i(void *object, const char *name, ...);    // deprecated, use polyglot_as_i32(polyglot_invoke(...))
-long truffle_invoke_l(void *object, const char *name, ...);   // deprecated, use polyglot_as_i64(polyglot_invoke(...))
-char truffle_invoke_c(void *object, const char *name, ...);   // deprecated, use polyglot_as_i8(polyglot_invoke(...))
-float truffle_invoke_f(void *object, const char *name, ...);  // deprecated, use polyglot_as_float(polyglot_invoke(...))
-double truffle_invoke_d(void *object, const char *name, ...); // deprecated, use polyglot_as_double(polyglot_invoke(...))
-bool truffle_invoke_b(void *object, const char *name, ...);   // deprecated, use polyglot_as_boolean(polyglot_invoke(...))
-
-// GetSize
-int truffle_get_size(const void *object); // renamed to polyglot_get_array_size
-
-// Unbox
-int truffle_unbox_i(void *object);    // renamed to polyglot_as_i32
-long truffle_unbox_l(void *object);   // renamed to polyglot_as_i64
-char truffle_unbox_c(void *object);   // renamed to polyglot_as_i8
-float truffle_unbox_f(void *object);  // renamed to polyglot_as_float
-double truffle_unbox_d(void *object); // renamed to polyglot_as_double
-bool truffle_unbox_b(void *object);   // renamed to polyglot_as_boolean
-
-// Read
-void *truffle_read(void *object, const char *name);    // renamed to polyglot_get_member
-int truffle_read_i(void *object, const char *name);    // deprecated, use polyglot_as_i32(polyglot_get_member(...))
-long truffle_read_l(void *object, const char *name);   // deprecated, use polyglot_as_i64(polyglot_get_member(...))
-char truffle_read_c(void *object, const char *name);   // deprecated, use polyglot_as_i8(polyglot_get_member(...))
-float truffle_read_f(void *object, const char *name);  // deprecated, use polyglot_as_float(polyglot_get_member(...))
-double truffle_read_d(void *object, const char *name); // deprecated, use polyglot_as_double(polyglot_get_member(...))
-bool truffle_read_b(void *object, const char *name);   // deprecated, use polyglot_as_boolean(polyglot_get_member(...))
-
-void *truffle_read_idx(void *object, int idx);    // renamed to polyglot_get_array_element
-int truffle_read_idx_i(void *object, int idx);    // deprecated, use polyglot_as_i32(polyglot_get_array_element(...))
-long truffle_read_idx_l(void *object, int idx);   // deprecated, use polyglot_as_i64(polyglot_get_array_element(...))
-char truffle_read_idx_c(void *object, int idx);   // deprecated, use polyglot_as_i8(polyglot_get_array_element(...))
-float truffle_read_idx_f(void *object, int idx);  // deprecated, use polyglot_as_float(polyglot_get_array_element(...))
-double truffle_read_idx_d(void *object, int idx); // deprecated, use polyglot_as_double(polyglot_get_array_element(...))
-bool truffle_read_idx_b(void *object, int idx);   // deprecated, use polyglot_as_boolean(polyglot_get_array_element(...))
-
-// Write
-void truffle_write(void *object, const char *name, void *value);    // renamed to polyglot_put_member
-void truffle_write_i(void *object, const char *name, int value);    // deprecated, use polyglot_put_member
-void truffle_write_l(void *object, const char *name, long value);   // deprecated, use polyglot_put_member
-void truffle_write_c(void *object, const char *name, char value);   // deprecated, use polyglot_put_member
-void truffle_write_f(void *object, const char *name, float value);  // deprecated, use polyglot_put_member
-void truffle_write_d(void *object, const char *name, double value); // deprecated, use polyglot_put_member
-void truffle_write_b(void *object, const char *name, bool value);   // deprecated, use polyglot_put_member
-
-void truffle_write_idx(void *object, int idx, void *value);    // renamed to polyglot_set_array_element
-void truffle_write_idx_i(void *object, int idx, int value);    // deprecated, use polyglot_set_array_element
-void truffle_write_idx_l(void *object, int idx, long value);   // deprecated, use polyglot_set_array_element
-void truffle_write_idx_c(void *object, int idx, char value);   // deprecated, use polyglot_set_array_element
-void truffle_write_idx_f(void *object, int idx, float value);  // deprecated, use polyglot_set_array_element
-void truffle_write_idx_d(void *object, int idx, double value); // deprecated, use polyglot_set_array_element
-void truffle_write_idx_b(void *object, int idx, bool value);   // deprecated, use polyglot_set_array_element
-
-// Strings
-void *truffle_read_string(const char *string);              // deprecated, use polyglot_from_string instead
-void *truffle_read_n_string(const char *string, int n);     // deprecated, use polyglot_from_string_n instead
-void *truffle_read_bytes(const char *bytes);                // deprecated, no replacement
-void *truffle_read_n_bytes(const char *bytes, int n);       // deprecated, no replacement
-const char *truffle_string_to_cstr(const char *string);     // deprecated, use polyglot_as_string instead
-void truffle_free_cstr(const char *truffle_allocated_cstr); // deprecated, no replacement
-
-void *truffle_sulong_function_to_native_pointer(void *sulongFunctionPointer, const void *signature); // deprecated, no replacement
-
-void *truffle_polyglot_eval(const char *mimeType, const char *code); // deprecated, use polyglot_eval instead
-
 #if defined(__cplusplus)
 }
 #endif

graalpython/com.oracle.graal.python.cext/modules/_mmap.c

@@ -0,0 +1,92 @@
+/* Copyright (c) 2019, Oracle and/or its affiliates.
+ * Copyright (C) 1996-2017 Python Software Foundation
+ *
+ * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+ */
+
+#include "../src/capi.h"
+
+typedef enum
+{
+    ACCESS_DEFAULT,
+    ACCESS_READ,
+    ACCESS_WRITE,
+    ACCESS_COPY
+} access_mode;
+
+typedef struct {
+    PyObject_HEAD
+    char *      data;
+    Py_ssize_t  size;
+    Py_ssize_t  pos;    /* relative to offset */
+#ifdef MS_WINDOWS
+    long long offset;
+#else
+    off_t       offset;
+#endif
+    int     exports;
+
+#ifdef MS_WINDOWS
+    HANDLE      map_handle;
+    HANDLE      file_handle;
+    char *      tagname;
+#endif
+
+#ifdef UNIX
+    int fd;
+#endif
+
+    PyObject *weakreflist;
+    access_mode access;
+} mmap_object;
+
+
+POLYGLOT_DECLARE_TYPE(mmap_object);
+static PyTypeObject mmap_object_type = PY_TRUFFLE_TYPE("mmap.mmap", NULL, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, sizeof(mmap_object));
+
+int mmap_getbuffer(PyObject *self, Py_buffer *view, int flags) {
+	// TODO(fa) readonly flag
+    return PyBuffer_FillInfo(view, (PyObject*)self, ((mmap_object *)self)->data, PyObject_Size((PyObject *)self), 0, flags);
+}
+
+static PyObject* mmap_init_bufferprotocol(PyObject* self, PyObject* mmap_type) {
+	assert(PyType_Check(mmap_type));
+    initialize_type_structure(&mmap_object_type, mmap_type, polyglot_mmap_object_typeid());
+
+	polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java(mmap_type), (getbufferproc) mmap_getbuffer, (releasebufferproc) NULL);
+	return Py_None;
+}
+
+static struct PyMethodDef module_functions[] = {
+    {"init_bufferprotocol", mmap_init_bufferprotocol, METH_O, NULL},
+    {NULL,       NULL}          /* sentinel */
+};
+
+static struct PyModuleDef mmapmodule = {
+    PyModuleDef_HEAD_INIT,
+    "_mmap",
+    NULL,
+    -1,
+    module_functions,
+    NULL,
+    NULL,
+    NULL,
+    NULL
+};
+
+PyMODINIT_FUNC
+PyInit__mmap(void)
+{
+    PyObject *dict, *module;
+
+    module = PyModule_Create(&mmapmodule);
+    if (module == NULL) {
+        return NULL;
+    }
+    dict = PyModule_GetDict(module);
+    if (!dict) {
+        return NULL;
+    }
+
+    return module;
+}