Merge commit '9cab1333a0b3aedb922b0b3b31c25cebc879b6fd' into release/graal-vm/1.0

Author: ansalond

3rd_party_licenses.txt

@@ -3,6 +3,23 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 1.0.0 RC10
+
+* Improve performance of C API upcalls
+* Improve performance of classmethods, staticmethods, `globals()`, and `locals()`
+* Improve performance of various string and bytes operations
+* Initial support for the `_thread` builtin module (actual multi-threading is still disabled, the API defaults to a dummy implementation)
+* Implement the `zipimporter` module
+* Support assignment to `object.__class__`
+* Use the new Truffle filesystem API to get/set the current working directory
+* Attempt our best to report side-effects in KEY_INFO
+* The KEYS message now responds with attributes and methods, never dict keys
+* Support the `input` builtin
+* Add DEBUG launcher options for performance debugging
+* Ensure context isolation for file descriptors and child PIDs
+* Fix passing custom locals and globals through `exec` and `eval`
+* Fixes to builtin `help`
+
 ## Version 1.0.0 RC9
 
 * Support `help` in the builtin Python shell

CHANGELOG.md

@@ -1,5 +1,5 @@
 {
-  overlay: "2b63e24614bc48b008ea15460ab22129f4a2c442",
+  overlay: "ffde557d62fff381069838b745c31b79c90e6e2a",
 
   // ======================================================================================================
   // 
@@ -70,7 +70,7 @@
       "git": ">=1.8.3",
       "mercurial": ">=3.2.4",
       "gcc": "==4.9.1",
-      "llvm": ">=4.0",
+      "llvm": "==4.0.1",
       "python": "==3.4.1",
       "libffi": ">=3.2.1",
       "bzip2": ">=1.0.6",

ci.jsonnet

@@ -145,9 +145,12 @@ Returns true for None only.
 
 ###### HAS_SIZE
 According to the Truffle interop contract answering `true` to `HAS_SIZE` implies
-that indexed element access is available. Thus, we answer `true` here only for
-(sub-)instances of `tuple`, `list`, `array.array`, `bytearray`, `bytes`, `str`,
-and `range`.
+that indexed element access is available. However, we cannot fully guarantee
+this. We may answer `true` here when the object has both a `__len__` field and a
+`__getitem__` field. If the object's length is reported >0, we also try to read
+the item `0` and if that fails, we answer `false`. If the object reports it's
+empty, we cannot know if a read with an index will actually work, but we'll
+report `true`.
 
 ###### GET_SIZE
 Calls `__len__`. Just because `GET_SIZE` returns something positive does not
@@ -159,27 +162,35 @@ knowing what the `__getitem__` method does with an integer argument. Use
 Returns true for those values that can be unboxed using the `UNBOX` message.
 
 ###### KEY_INFO
-This will lookup the key using `READ`, assume it is readable and writable, and
-check `IS_EXECUTABLE`.
+This will lookup the key using the Python MRO. It will check if it's readable
+and writable, and also check if it has side-effects based on wether it is an
+inherited descriptor (i.e., an object with `__get__`, `__set__`, and/or
+`__delete__`). If the owner of the key is mutable (the owner being the class the
+key is inherited from or the object itself) then `REMOVABLE` and `MODIFABLE` are
+true. If the object itself is mutable, `INSERTABLE` will also be true. Finally,
+if the attribute is a function or it is *not* a descriptor and has a `__call__`,
+we declare it `INOCABLE`. We don't do this for descriptors, because we would
+have to run the `__get__` method and this message should not have side-effects.
 
 ###### HAS_KEYS
-Returns true for any boxed Python object, so small integers, booleans, or floats
-usually don't return true.
+Always returns true.
 
 ###### KEYS
-This returns the direct attributes of the receiver object, which would usually
-be available through `__getattribute__`. 
-
-The `KEYS` message requires the returned object to have only `java.lang.String`
-items. If the object responds to `keys`, `values`, `items`, and `__getitem__`,
-we assume it is Mapping, and we present the result of the `keys` method in
-combination with the attributes if, and only if, all keys are strings. This is
-roughly parallel to how `READ` and `WRITE` would be handled for string keys.
+This returns the all attributes of the receiver object that would usually be
+available through `__getattribute__`, i.e., both inherited and direct
+attributes.
+
+If the object responds to `keys`, `values`, `items`, and `__getitem__`, we
+assume it is Mapping, and we present the String result of the `keys` method in
+combination with the attributes, prefixed with `[` if, and only if, the request
+asked for _internal_ keys also. The `KEYS` message requires the returned object
+to have only `java.lang.String` items, so inlo String keys are added to the
+result set. The `[` prefix ensures that in our handling of `READ` and `WRITE`
+messages we also treat them as mapping entries, not attributes.
 
 It's still possible that none of the keys can be `READ`: the `READ` message uses
 Python semantics for lookup, which means that an inherited descriptor with a
-`__get__` method may still come before the object's keys and do anything
-(including raising an `AttributeError`).
+`__get__` method or the `__getitem__` method may still intercept actual access.
 
 ###### IS_POINTER
 Returns true if the object is a Python function defined in a Python C extension

doc/INTEROP.md

@@ -55,6 +55,109 @@
 ATTR_TEARDOWN = '__teardown__'
 
 
+# ----------------------------------------------------------------------------------------------------------------------
+#
+# the CUSUM method adapted for warmup detection within a given threshold (initial iterations)
+#
+# ----------------------------------------------------------------------------------------------------------------------
+def zeros(n):
+    return [0 for _ in range(n)]
+
+
+def append(arr, val):
+    if isinstance(arr, list):
+        return arr + [val]
+    else:
+        return [val] + arr
+
+
+def cusum(values, threshold=1.0, drift=0.0):
+    csum_pos, csum_neg = zeros(len(values)), zeros(len(values))
+    change_points = []
+    for i in range(1, len(values)):
+        diff = values[i] - values[i - 1]
+        csum_pos[i] = csum_pos[i-1] + diff - drift
+        csum_neg[i] = csum_neg[i-1] - diff - drift
+
+        if csum_pos[i] < 0:
+            csum_pos[i] = 0
+        if csum_neg[i] < 0:
+            csum_neg[i] = 0
+
+        if csum_pos[i] > threshold or csum_neg[i] > threshold:
+            change_points = append(change_points, i)
+            csum_pos[i], csum_neg[i] = 0, 0
+
+    return change_points
+
+
+def avg(values):
+    return float(sum(values)) / len(values)
+
+
+def norm(values):
+    _max, _min  = max(values), min(values)
+    return [float(v - _min) / (_max - _min) * 100.0 for v in values]
+
+
+def pairwise_slopes(values, cp):
+    return [abs(float(values[i+1] - values[i]) / float(cp[i+1] - cp[i])) for i in range(len(values)-1)]
+
+
+def last_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    end_runs_idx = len(values) - int(len(values) * n)
+    end_runs_idx = len(values) - 1 if end_runs_idx >= len(values) else end_runs_idx
+    return values[end_runs_idx:], list(range(end_runs_idx, len(values)))
+
+
+def first_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    first_run_idx = int(len(values) * n)
+    return first_run_idx -1 if first_run_idx == len(values) else first_run_idx
+
+
+def detect_warmup(values, cp_threshold=0.03, stability_slope_grade=0.01):
+    """
+    detect the point of warmup point (iteration / run)
+
+    :param values: the durations for each run
+    :param cp_threshold:  the percent in value difference for a point to be considered a change point (percentage)
+    :param stability_slope_grade: the slope grade (percentage). A grade of 1% corresponds to a slope of 0.5 degrees
+    :return: the change point or -1 if not detected
+    """
+    # normalize all
+    stability_slope_grade *= 100.0
+    cp_threshold *= 100
+    values = norm(values)
+
+    try:
+        cp = cusum(values, threshold=cp_threshold)
+        rolling_avg = [avg(values[i:]) for i in cp]
+
+        def warmup(cp_index):
+            val_idx = cp[cp_index] + 1
+            return val_idx if val_idx < len(values) else -1
+
+        # find the point where the duration avg is below the cp threshold
+        for i, d in enumerate(rolling_avg):
+            if d <= cp_threshold:
+                return warmup(i)
+
+        # could not find something below the CP threshold (noise in the data), use the stabilisation of slopes
+        last_n_vals, last_n_idx = last_n_percent_runs(values, 0.1)
+        slopes = pairwise_slopes(rolling_avg + last_n_vals, cp + last_n_idx)
+
+        for i, d in enumerate(slopes):
+            if d <= stability_slope_grade:
+                return warmup(i)
+
+        return -1
+    except Exception as e:
+        print("exception occurred while detecting warmup: %s" % e)
+        return -1
+
+
 def ccompile(name, code):
     from importlib import invalidate_caches
     from distutils.core import setup, Extension
@@ -89,7 +192,14 @@ def _as_int(value):
 
 
 class BenchRunner(object):
-    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
+    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0):
+        assert isinstance(iterations, int), \
+            "BenchRunner iterations argument must be an int, got %s instead" % iterations
+        assert isinstance(warmup, int), \
+            "BenchRunner warmup argument must be an int, got %s instead" % warmup
+        assert isinstance(warmup_runs, int), \
+            "BenchRunner warmup_runs argument must be an int, got %s instead" % warmup_runs
+
         if bench_args is None:
             bench_args = []
         self.bench_module = BenchRunner.get_bench_module(bench_file)
@@ -98,10 +208,8 @@ def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
         _iterations = _as_int(iterations)
         self._run_once = _iterations <= 1
         self.iterations = 1 if self._run_once else _iterations
-
-        assert isinstance(self.iterations, int)
-        self.warmup = _as_int(warmup)
-        assert isinstance(self.warmup, int)
+        self.warmup_runs = warmup_runs if warmup_runs > 0 else 0
+        self.warmup = warmup if warmup > 0 else -1
 
     @staticmethod
     def get_bench_module(bench_file):
@@ -139,9 +247,10 @@ def _call_attr(self, attr_name, *args):
 
     def run(self):
         if self._run_once:
-            print("### %s, exactly one iteration (no warmup curves)" % (self.bench_module.__name__))
+            print("### %s, exactly one iteration (no warmup curves)" % self.bench_module.__name__)
         else:
-            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__, self.warmup, self.iterations))
+            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__,
+                                                                          self.warmup_runs, self.iterations))
 
         # process the args if the processor function is defined
         args = self._call_attr(ATTR_PROCESS_ARGS, *self.bench_args)
@@ -159,9 +268,9 @@ def run(self):
         bench_func = self._get_attr(ATTR_BENCHMARK)
         durations = []
         if bench_func and hasattr(bench_func, '__call__'):
-            if self.warmup:
-                print("### warming up for %s iterations ... " % self.warmup)
-                for _ in range(self.warmup):
+            if self.warmup_runs:
+                print("### (pre)warming up for %s iterations ... " % self.warmup_runs)
+                for _ in range(self.warmup_runs):
                     bench_func(*args)
 
             for iteration in range(self.iterations):
@@ -173,21 +282,46 @@ def run(self):
                 if self._run_once:
                     print("@@@ name=%s, duration=%s" % (self.bench_module.__name__, duration_str))
                 else:
-                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__, duration_str))
+                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__,
+                                                                      duration_str))
 
         print(_HRULE)
         print("### teardown ... ")
         self._call_attr(ATTR_TEARDOWN)
         print("### benchmark complete")
         print(_HRULE)
-        print("### BEST     duration: %.3f s" % min(durations))
-        print("### WORST    duration: %.3f s" % max(durations))
-        print("### AVG      duration: %.3f" % (sum(durations) / len(durations)))
+
+        # summary
+        if self._run_once:
+            print("### SINGLE RUN        duration: %.3f s" % durations[0])
+        else:
+            print("### BEST                duration: %.3f s" % min(durations))
+            print("### WORST               duration: %.3f s" % max(durations))
+            print("### AVG (all runs)      duration: %.3f s" % (sum(durations) / len(durations)))
+            warmup_iter = self.warmup if self.warmup > 0 else detect_warmup(durations)
+            # if we cannot detect a warmup starting point but we performed some pre runs, we take a starting point
+            # after the 10% of the first runs ...
+            if warmup_iter < 0 and self.warmup_runs > 0:
+                print("### warmup could not be detected, but %s pre-runs were executed.\n"
+                      "### we assume the benchmark is warmed up and pick an iteration "
+                      "in the first 10%% of the runs" % self.warmup_runs)
+                warmup_iter = first_n_percent_runs(durations, 0.1)
+
+            if warmup_iter > 0:
+                print("### WARMUP %s at iteration: %d" % ("specified" if self.warmup > 0 else "detected", warmup_iter))
+                no_warmup_durations = durations[warmup_iter:]
+                print("### AVG (no warmup)     duration: %.3f s" % (sum(no_warmup_durations) / len(no_warmup_durations)))
+            else:
+                print("### WARMUP iteration not specified or could not be detected")
+
+        print(_HRULE)
+        print("### RAW DURATIONS: %s" % str(durations))
         print(_HRULE)
 
 
 def run_benchmark(args):
-    warmup = 0
+    warmup = -1
+    warmup_runs = 0
     iterations = 1
     bench_file = None
     bench_args = []
@@ -208,6 +342,12 @@ def run_benchmark(args):
         elif arg.startswith("--warmup"):
             warmup = _as_int(arg.split("=")[1])
 
+        elif arg == '-r':
+            i += 1
+            warmup_runs = _as_int(args[i])
+        elif arg.startswith("--warmup-runs"):
+            warmup_runs = _as_int(arg.split("=")[1])
+
         elif arg == '-p':
             i += 1
             paths = args[i].split(",")
@@ -229,7 +369,7 @@ def run_benchmark(args):
     else:
         print("### no extra module search paths specified")
 
-    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup).run()
+    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs).run()
 
 
 if __name__ == '__main__':

graalpython/benchmarks/src/harness.py

@@ -54,6 +54,9 @@ void *truffle_assign_managed(void *dst, void *managed);
 void *truffle_deref_handle_for_managed(void *managed);
 bool truffle_cannot_be_handle(void *nativeHandle);
 
+// wrapping functions
+void *truffle_decorate_function(void *function, void *wrapper);
+
 /*
  * All function below here are deprecated and will be removed in a future release.
  * Use the equivalent functions from <polyglot.h> instead.