Merge remote-tracking branch 'origin/master' into hotfix/null-python-home

Author: ansalond

CHANGELOG.md

@@ -3,6 +3,15 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 1.0.0 RC13
+
+* Support marshal.dumps and marshal.loads for code objects and some other built-in objects
+* Fix installation of NumPy in a venv
+* Initial support for module mmap
+* Support debugging with workspace files in the Chrome debugger
+* Support the PEP 553 breakpoint() message
+* Support running weak reference callbacks and signals on the main thread
+
 ## Version 1.0.0 RC12
 
 * Support the `__class__` variable in the class scope

ci.jsonnet

@@ -1,5 +1,5 @@
 {
-  overlay: "e08ca6c848943a04c5e46d54272ff927ff3d51f7",
+  overlay: "d20cc2abdeb3cfb022e1a8035e40e350e5cfe5fc",
 
   // ======================================================================================================
   // 

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -74,6 +74,9 @@
 PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
 PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
 PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
+PTRN_REMOTE_HOST = re.compile(r"(?P<user>\w+)@(?P<host>[\w.]+):(?P<path>.+)")
+PTRN_VALID_CSV_NAME = re.compile(r"unittests-\d{4}-\d{2}-\d{2}.csv")
+
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
@@ -192,6 +195,27 @@ def _sorter(iterable):
     return unittests[:limit] if limit else unittests
 
 
+def get_remote_host(scp_path):
+    match = re.match(PTRN_REMOTE_HOST, scp_path)
+    return match.group('user'), match.group('host'), match.group('path')
+
+
+def ssh_ls(scp_path):
+    user, host, path = get_remote_host(scp_path)
+    cmd = ['ssh', '{}@{}'.format(user, host), 'ls', '-l', path]
+    return map(lambda l: l.split()[-1], _run_cmd(cmd)[1].splitlines())
+
+
+def read_csv(path):
+    rows = []
+    with open(path, "r") as CSV_FILE:
+        reader = csv.reader(CSV_FILE)
+        headers = next(reader)[1:]
+        for row in reader:
+            rows.append(row)
+    return rows
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # result (output processing)
@@ -597,7 +621,7 @@ def format_val(row, k):
             elif k == Col.NUM_TESTS:
                 _class = "text-dark"
             else:
-                _class = "text-danger" if value < 0 else "text-muted"
+                _class = "text-danger" if value and value < 0 else "text-muted"
             return '<span class="{} h6"><b>{}</b></span>'.format(_class, value)
 
         _tbody = '\n'.join([
@@ -694,6 +718,10 @@ def main(prog, args):
     parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
     parser.add_argument("-s", "--skip_tests", help="Run all unittests except (comma sep values)."
                                                    "the only_tets option takes precedence", default=None)
+    parser.add_argument("-r", "--regression_running_tests", help="Regression threshold for running tests.", type=float,
+                        default=None)
+    parser.add_argument("-g", "--gate", help="Run in gate mode (Skip cpython runs; Do not upload results; "
+                                             "Detect regressions).", action="store_true")
     parser.add_argument("path", help="Path to store the csv output and logs to.", nargs='?', default=None)
 
     global flags
@@ -708,6 +736,12 @@ def main(prog, args):
     else:
         log("[INFO] results will not be saved remotely")
 
+    if flags.gate:
+        log("[INFO] running in gate mode")
+        if not flags.regression_running_tests:
+            log("[WARNING] --regression_running_tests not set while in gate mode. "
+                "Regression detection will not be performed")
+
     def _fmt(t):
         t = t.strip()
         return os.path.join(flags.tests_path, t if t.endswith(".py") else t + ".py")
@@ -720,15 +754,22 @@ def _fmt(t):
         unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
 
     # get cpython stats
-    log(HR)
-    log("[INFO] get cpython stats")
-    cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
-    cpy_stats = process_output('\n'.join(cpy_results))[-1]
+    if not flags.gate:
+        log(HR)
+        log("[INFO] get cpython stats")
+        cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
+        cpy_stats = process_output('\n'.join(cpy_results))[-1]
+        # handle the timeout
+        timeout = flags.timeout if flags.timeout else None
+    else:
+        cpy_stats = None
+        # handle the timeout
+        timeout = flags.timeout if flags.timeout else 60 * 5  # 5 minutes if no value specified (in gate mode only)
 
     # get graalpython stats
     log(HR)
     log("[INFO] get graalpython stats")
-    results = run_unittests(unittests, flags.timeout, with_cpython=False)
+    results = run_unittests(unittests, timeout, with_cpython=False)
     txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
     output = save_as_txt(txt_report_path, results)
 
@@ -749,15 +790,44 @@ def _fmt(t):
     log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
 
     html_report_path = file_name(HTML_RESULTS_NAME, current_date)
-    save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
+    if not flags.gate:
+        save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
 
-    if flags.path:
+    if not flags.gate and flags.path:
         log("[SAVE] saving results to {} ... ", flags.path)
         scp(txt_report_path, flags.path)
         scp(csv_report_path, flags.path)
         scp(html_report_path, flags.path)
 
+    gate_failed = False
+    if flags.gate and flags.regression_running_tests:
+        log("[REGRESSION] detecting regression, acceptance threshold = {}%".format(
+            flags.regression_running_tests * 100))
+        csv_files = list(filter(lambda entry: True if PTRN_VALID_CSV_NAME.match(entry) else False, ssh_ls(flags.path)))
+        last_csv = csv_files[-1]
+        # log('\n'.join(csv_files))
+        # read the remote csv and extract stats
+        log("[REGRESSION] comparing against: {}".format(last_csv))
+        scp('{}/{}'.format(flags.path, last_csv), '.', destination_name=last_csv)
+        rows = read_csv(last_csv)
+        prev_totals = {
+            Col.NUM_TESTS: int(rows[-1][1]),
+            Col.NUM_FAILS: int(rows[-1][2]),
+            Col.NUM_ERRORS: int(rows[-1][3]),
+            Col.NUM_SKIPPED: int(rows[-1][4]),
+            Col.NUM_PASSES: int(rows[-1][5]),
+        }
+        print(prev_totals)
+        if float(totals[Col.NUM_TESTS]) < float(prev_totals[Col.NUM_TESTS]) * (1.0 - flags.regression_running_tests):
+            log("[REGRESSION] REGRESSION DETECTED, passed {} tests vs {} from {}".format(
+                totals[Col.NUM_TESTS], prev_totals[Col.NUM_TESTS], last_csv))
+            gate_failed = True
+        else:
+            log("[REGRESSION] no regression detected")
+
     log("[DONE]")
+    if flags.gate and gate_failed:
+        exit(1)
 
 
 if __name__ == "__main__":

graalpython/com.oracle.graal.python.test/src/tests/test_itertools.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2019, Oracle and/or its affiliates.
+# Copyright (C) 1996-2017 Python Software Foundation
+#
+# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+
+import unittest
+from itertools import *
+
+class CombinationsTests(unittest.TestCase):
+
+    def test_combinations_with_replacement(self):
+        cwr = combinations_with_replacement
+        self.assertRaises(TypeError, cwr, 'abc')       # missing r argument
+        self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
+        self.assertRaises(TypeError, cwr, None)        # pool is not iterable
+        self.assertRaises(ValueError, cwr, 'abc', -2)  # r is negative
+
+        result = list()
+        for a in cwr('ABC', 2):
+            result += a
+        correct = [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]
+        compare = list();
+        for a in correct:
+            compare += a;
+        self.assertEqual(result,compare)
+

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TruffleCextBuiltins.java

@@ -1430,7 +1430,7 @@ Object doUnpackLong(long start, long stop, long step, long length) {
         }
 
         @Specialization(replaces = {"doUnpackLong", "doUnpack"})
-        Object doUnpackLongOvf(long start, long stop, long step, long length) {
+        Object doUnpackLongOvf0(long start, long stop, long step, long length) {
             try {
                 PSlice tmpSlice = factory().createSlice(PInt.intValueExact(start), PInt.intValueExact(stop), PInt.intValueExact(step));
                 SliceInfo actualIndices = tmpSlice.computeIndices(length > Integer.MAX_VALUE ? Integer.MAX_VALUE : PInt.intValueExact(length));
@@ -1440,8 +1440,8 @@ Object doUnpackLongOvf(long start, long stop, long step, long length) {
             }
         }
 
-        @Specialization(replaces = {"doUnpackLongOvf"})
-        Object doUnpackLongOvf(Object start, Object stop, Object step, Object lengthObj,
+        @Specialization(replaces = {"doUnpackLongOvf0"})
+        Object doUnpackLongOvf1(Object start, Object stop, Object step, Object lengthObj,
                         @Cached("createOverflow()") CastToIndexNode castToIndexNode,
                         @Cached("create()") IsBuiltinClassProfile profile,
                         @Cached("create()") SliceLiteralNode sliceLiteralNode) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java

@@ -230,24 +230,24 @@ protected Object doItUnboxedIndirect(VirtualFrame frame, @SuppressWarnings("unus
         }
 
         @Specialization(limit = "getCallSiteInlineCacheMaxDepth()", guards = {"self == cachedSelf"})
-        protected Object doIt(VirtualFrame frame, @SuppressWarnings("unused") PythonAbstractClass self, Object[] arguments, PKeyword[] keywords,
+        protected Object doIt0(VirtualFrame frame, @SuppressWarnings("unused") PythonAbstractClass self, Object[] arguments, PKeyword[] keywords,
                         @Cached("self") PythonAbstractClass cachedSelf) {
             return op(frame, cachedSelf, arguments, keywords, true);
         }
 
-        @Specialization(replaces = "doIt")
-        protected Object doItIndirect(VirtualFrame frame, PythonAbstractClass self, Object[] arguments, PKeyword[] keywords) {
+        @Specialization(replaces = "doIt0")
+        protected Object doItIndirect0(VirtualFrame frame, PythonAbstractClass self, Object[] arguments, PKeyword[] keywords) {
             return op(frame, self, arguments, keywords, true);
         }
 
         @Specialization(limit = "getCallSiteInlineCacheMaxDepth()", guards = {"self == cachedSelf"})
-        protected Object doIt(VirtualFrame frame, @SuppressWarnings("unused") PythonNativeObject self, Object[] arguments, PKeyword[] keywords,
+        protected Object doIt1(VirtualFrame frame, @SuppressWarnings("unused") PythonNativeObject self, Object[] arguments, PKeyword[] keywords,
                         @Cached("self") PythonNativeObject cachedSelf) {
             return op(frame, PythonNativeClass.cast(cachedSelf), arguments, keywords, true);
         }
 
-        @Specialization(replaces = "doIt")
-        protected Object doItIndirect(VirtualFrame frame, PythonNativeObject self, Object[] arguments, PKeyword[] keywords) {
+        @Specialization(replaces = "doIt1")
+        protected Object doItIndirect1(VirtualFrame frame, PythonNativeObject self, Object[] arguments, PKeyword[] keywords) {
             return op(frame, PythonNativeClass.cast(self), arguments, keywords, true);
         }
 

graalpython/lib-graalpython/itertools.py

@@ -523,7 +523,7 @@ class combinations():
 
     def __init__(self, pool, indices, r):
         self.pool = pool
-        self.indices = range(indices)
+        self.indices = indices
         if r < 0:
             raise ValueError("r must be non-negative")
         self.r = r
@@ -593,7 +593,7 @@ def __init__(self, iterable, r):
         if r < 0:
             raise ValueError("r must be non-negative")
         indices = [0] * r
-        combinations.__init__(pool, indices, r)
+        super().__init__(pool, indices, r)
         self.stopped = len(pool) == 0 and r > 0
 
     def get_maximum(self, i):

graalpython/lib-python/3/venv/__init__.py

@@ -135,7 +135,7 @@ def create_if_needed(d):
             if sys.platform == "win32":
                 f.write(" %*")
             else:
-                f.write(" $@")
+                f.write(" \"$@\"")
 
         if sys.platform != "win32":
             os.chmod(script, 0o777)

mx.graalpython/copyrights/overrides

@@ -233,6 +233,7 @@ graalpython/com.oracle.graal.python.test/src/tests/test_getattribute-bimorphic-i
 graalpython/com.oracle.graal.python.test/src/tests/test_getattribute-bimorphic-inobject.py,zippy.copyright
 graalpython/com.oracle.graal.python.test/src/tests/test_if-class-none.py,zippy.copyright
 graalpython/com.oracle.graal.python.test/src/tests/test_if.py,zippy.copyright
+graalpython/com.oracle.graal.python.test/src/tests/test_itertools.py,python.copyright
 graalpython/com.oracle.graal.python.test/src/tests/test_list.py,python.copyright
 graalpython/com.oracle.graal.python.test/src/tests/test_builtin.py,python.copyright
 graalpython/com.oracle.graal.python.test/src/tests/test_mandelbrot3.py,benchmarks.copyright