Merge pull request #3 from ericsnowcurrently/pyperformance

Use pyperformance to run the benchmarks.

Author: kmod

.gitignore

@@ -129,3 +129,4 @@ dmypy.json
 .pyre/
 
 results
+benchmarks/bm_pytorch_alexnet_inference/data/dog.jpg

README.md

@@ -1,2 +1,21 @@
 # python-macrobenchmarks
 A collection of macro benchmarks for the Python programming language
+
+
+## usage
+
+```shell
+# Run the default benchmarks:
+python3 -m pyperformance run --manifest $PWD/benchmarks/MANIFEST
+```
+
+The benchmarks can still be run without pyperformance.  This will produce
+ the old results format.
+
+```shell
+# Run the benchmarks:
+sh ./run_all.sh
+
+# Run the mypy benchmark using mypyc:
+sh ./run_mypy.sh
+```

benchmarks/.libs/legacyutils.py

@@ -0,0 +1,22 @@
+import json
+import sys
+
+
+def maybe_handle_legacy(bench_func, *args, loopsarg='loops', legacyarg=None):
+    if '--legacy' not in sys.argv:
+        return
+    argv = list(sys.argv[1:])
+    argv.remove('--legacy')
+
+    kwargs = {}
+    if legacyarg:
+        kwargs[legacyarg] = True
+    if argv:
+        assert loopsarg
+        kwargs[loopsarg] = int(argv[0])
+
+    _, times = bench_func(*args, **kwargs)
+    if len(argv) > 1:
+        json.dump(times, open(argv[1], 'w'))
+
+    sys.exit(0)

benchmarks/.libs/netutils.py

@@ -0,0 +1,88 @@
+import contextlib
+import ipaddress
+import os.path
+import socket
+import subprocess
+import time
+
+
+@contextlib.contextmanager
+def serving(argv, sitedir, addr, *,
+            pause=None,
+            kill=False,
+            quiet=True,
+            ):
+    if os.path.exists(addr):
+        sock = addr
+        addr = None
+        try:
+            os.remove(sock)
+        except FileNotFoundError:
+            pass
+    else:
+        sock = None
+
+    p = subprocess.Popen(
+        argv,
+        cwd=sitedir,
+        stdout=subprocess.DEVNULL if quiet else None,
+        stderr=subprocess.STDOUT if quiet else None,
+    )
+    try:
+        if pause:
+            time.sleep(pause)
+        if not sock:
+            try:
+                waitUntilUp(addr)
+            except NotImplementedError:
+                sock = addr
+                addr = None
+        if sock:
+            while not os.path.exists(sock):
+                time.sleep(0.001)
+        assert p.poll() is None, p.poll()
+        yield
+        assert p.poll() is None, p.poll()
+    finally:
+        p.terminate()
+        if kill:
+            p.kill()
+        p.wait()
+
+
+def waitUntilUp(addr, timeout=10.0):
+    end = time.time() + timeout
+    addr = parse_socket_addr(addr)
+    started = False
+    current = time.time()
+    while not started or current <= end:
+        try:
+            with socket.create_connection(addr) as sock:
+                return
+        except ConnectionRefusedError:
+            time.sleep(0.001)
+        started = True
+        current = time.time()
+    raise Exception('Timeout reached when trying to connect')
+
+
+def parse_socket_addr(addr, *, resolve=True):
+    if not isinstance(addr, str):
+        raise NotImplementedError(addr)
+    host, _, port = addr.partition(':')
+
+    if not host:
+        raise NotImplementedError(addr)
+    try:
+        host = ipaddress.ip_address(host)
+    except ValueError:
+        raise NotImplementedError(addr)
+    host = str(host)
+
+    if not port:
+        raise NotImplementedError(addr)
+    if not port.isdigit():
+        raise NotImplementedError(addr)
+    port = int(port)
+
+    return (host, port)

benchmarks/MANIFEST

@@ -0,0 +1,19 @@
+[benchmarks]
+
+name	metafile
+aiohttp	<local>
+djangocms	<local>
+flaskblogging	<local>
+gevent_hub	<local>
+gunicorn	<local>
+json	<local>
+kinto	<local>
+mypy	<local>
+mypyc	<local:mypy>
+pycparser	<local>
+pylint	<local>
+pytorch_alexnet_inference	<local>
+thrift	<local>
+
+[group default]
+-mypyc

benchmarks/aiohttp.py

@@ -0,0 +1 @@
+../pyproject.toml

benchmarks/base.toml

@@ -0,0 +1 @@
+../.libs/legacyutils.py

data/gunicorn_serve.py renamed to benchmarks/bm_aiohttp/data/serve.py

@@ -0,0 +1 @@
+../.libs/netutils.py