kinto (uwsgi+pyramid) benchmark

kinto is a project built on top of pyramid
run it using nginx+uwsgi

Author: kmod

benchmarks/kinto_bench.py

@@ -0,0 +1,75 @@
+import json
+import os
+import requests
+import subprocess
+import sys
+import threading
+import time
+import urllib
+
+from djangocms import waitUntilUp
+
+from os.path import join, abspath, dirname
+
+if __name__ == "__main__":
+    exe = sys.executable
+    def bin(name):
+        return join(dirname(exe), name)
+    def rel(path):
+        return abspath(join(dirname(__file__), path))
+
+    times = []
+
+    subprocess.check_call([abspath(exe), rel("../data/kinto_project/setup.py"), "develop"], cwd=rel("../data/kinto_project"), stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT)
+
+    try:
+        os.remove("/tmp/kinto.sock")
+    except FileNotFoundError:
+        pass
+    p1 = subprocess.Popen([bin("uwsgi"), rel("../data/kinto_project/production.ini")], cwd=rel("../data/kinto_project"), stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT)
+    # p1 = subprocess.Popen([bin("uwsgi"), rel("../data/kinto_project/production.ini")], cwd=rel("../data/kinto_project"))
+    while not os.path.exists("/tmp/kinto.sock"):
+        time.sleep(0.001)
+
+    # p2 = subprocess.Popen(["nginx", "-c", abspath("../data/kinto_project/nginx.conf"), "-p", abspath("../data/kinto_project")], cwd="../data/kinto_project", stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT)
+    p2 = subprocess.Popen(["nginx", "-c", rel("../data/kinto_project/nginx.conf"), "-p", rel("../data/kinto_project")], cwd=rel("../data/kinto_project"))
+
+    time.sleep(0.010)
+
+    try:
+        waitUntilUp(("127.0.0.1", 8000))
+
+        assert p1.poll() is None, p1.poll()
+        assert p2.poll() is None, p2.poll()
+
+        print(requests.get("http://localhost:8000/v1").text)
+        # print(requests.put("http://localhost:8000/v1/accounts/testuser", json={"data": {"password": "password1"}}).text)
+
+        n = 5000
+        if len(sys.argv) > 1:
+            n = int(sys.argv[1])
+
+        start = time.time()
+        for i in range(n):
+            times.append(time.time())
+            if i % 100 == 0:
+                print(i, time.time() - start)
+            # requests.get("http://localhost:8000/v1/").text
+            urllib.request.urlopen("http://localhost:8000/v1/").read()
+        times.append(time.time())
+        elapsed = time.time() - start
+        print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed))
+
+        assert p1.poll() is None, p1.poll()
+        assert p2.poll() is None, p2.poll()
+
+    finally:
+        p1.terminate()
+        p1.kill()
+        p1.wait()
+        # p2.kill()
+        p2.terminate()
+        p2.wait()
+
+    if len(sys.argv) > 2:
+        json.dump(times, open(sys.argv[2], 'w'))

benchmarks/kinto_bench_requirements.txt

@@ -0,0 +1,57 @@
+appdirs==1.4.3
+attrs==20.3.0
+bcrypt==3.2.0
+CacheControl==0.12.6
+certifi==2019.11.28
+cffi==1.14.5
+chardet==3.0.4
+colander==1.8.3
+colorama==0.4.3
+contextlib2==0.6.0
+cornice==5.2.0
+cornice-swagger==1.0.0
+distlib==0.3.0
+distro==1.4.0
+dockerflow==2020.10.0
+html5lib==1.0.1
+hupper==1.10.2
+idna==2.8
+ipaddr==2.2.0
+iso8601==0.1.14
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema==3.2.0
+kinto==14.2.0
+lockfile==0.12.2
+logging-color-formatter==1.0.2
+msgpack==0.6.2
+packaging==20.3
+PasteDeploy==2.1.1
+pep517==0.8.2
+plaster==1.0
+plaster-pastedeploy==0.7
+progress==1.5
+pycparser==2.20
+pyparsing==2.4.6
+pyramid==1.10.8
+pyramid-mailer==0.15.1
+pyramid-multiauth==0.9.0
+pyramid-tm==2.4
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+pytoml==0.1.21
+repoze.sendmail==4.4.1
+requests==2.22.0
+retrying==1.3.3
+six==1.14.0
+transaction==3.0.1
+translationstring==1.4
+ujson==4.0.2
+urllib3==1.25.8
+uWSGI==2.0.19.1
+venusian==3.0.0
+waitress==2.0.0
+webencodings==0.5.1
+WebOb==1.8.7
+zope.deprecation==4.4.0
+zope.interface==5.4.0

data/kinto_project/.coveragerc

@@ -0,0 +1,3 @@
+[run]
+source = kinto_project
+omit = kinto_project/test*

data/kinto_project/CHANGES.txt

@@ -0,0 +1,4 @@
+0.0
+---
+
+-  Initial version

data/kinto_project/MANIFEST.in

@@ -0,0 +1,2 @@
+include *.txt *.ini *.cfg *.rst
+recursive-include kinto_project *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml *.jinja2

data/kinto_project/README.txt

@@ -0,0 +1,12 @@
+kinto_project README
+==================
+
+Getting Started
+---------------
+
+- cd <directory containing this file>
+
+- $VENV/bin/pip install -e .
+
+- $VENV/bin/pserve development.ini
+

data/kinto_project/app.wsgi

@@ -0,0 +1,20 @@
+import configparser
+import logging.config
+import os
+
+from kinto import main
+
+here = os.path.dirname(__file__)
+
+ini_path = os.environ.get('KINTO_INI')
+if ini_path is None:
+    ini_path = os.path.join(here, 'config', 'kinto.ini')
+
+# Set up logging
+logging.config.fileConfig(ini_path)
+
+# Parse config and create WSGI app
+config = configparser.ConfigParser()
+config.read(ini_path)
+
+application = main(config.items('DEFAULT'), **dict(config.items('app:main')))

data/kinto_project/config/kinto.ini

@@ -0,0 +1,72 @@
+###
+# app configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/environment.html
+###
+
+[app:main]
+use = egg:kinto_project
+
+pyramid.reload_templates = false
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.default_locale_name = en
+
+###
+# wsgi server configuration
+###
+
+[server:main]
+use = egg:waitress#main
+listen = *:6543
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/logging.html
+###
+
+[loggers]
+keys = root, kinto_project
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_kinto_project]
+level = WARN
+handlers =
+qualname = kinto_project
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
+
+[uwsgi]
+wsgi-file = app.wsgi
+# virtualenv = .venv
+socket = /tmp/kinto.sock
+# uid = kinto
+# gid = kinto
+enable-threads = true
+chmod-socket = 666
+processes = 1
+master = true
+module = kinto
+harakiri = 120
+lazy = true
+lazy-apps = true
+single-interpreter = true
+buffer-size = 65535
+post-buffering = 65535
+plugin = python

data/kinto_project/development.ini

@@ -0,0 +1,78 @@
+###
+# app configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/environment.html
+###
+
+[app:main]
+use = egg:kinto_project
+
+pyramid.reload_templates = true
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.default_locale_name = en
+pyramid.includes =
+    pyramid_debugtoolbar
+
+# By default, the toolbar only appears for clients from IP addresses
+# '127.0.0.1' and '::1'.
+# debugtoolbar.hosts = 127.0.0.1 ::1
+
+###
+# wsgi server configuration
+###
+
+[server:main]
+use = egg:waitress#main
+listen = localhost:6543
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/logging.html
+###
+
+[loggers]
+keys = root, kinto_project
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_kinto_project]
+level = DEBUG
+handlers =
+qualname = kinto_project
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
+
+[uwsgi]
+wsgi-file = app.wsgi
+# virtualenv = .venv
+socket = /tmp/kinto.sock
+# uid = kinto
+# gid = kinto
+enable-threads = true
+chmod-socket = 666
+processes = 1
+master = true
+module = kinto
+harakiri = 120
+lazy = true
+lazy-apps = true
+single-interpreter = true
+buffer-size = 65535
+post-buffering = 65535
+plugin = python

data/kinto_project/kinto_project/__init__.py

@@ -0,0 +1,14 @@
+import pkg_resources
+
+import kinto.core
+from pyramid.config import Configurator
+
+# Module version, as defined in PEP-0396.
+__version__ = pkg_resources.get_distribution(__package__).version
+
+
+def main(global_config, **settings):
+    config = Configurator(settings=settings)
+
+    kinto.core.initialize(config, __version__, 'myproject')
+    return config.make_wsgi_app()