From d67c82a6e09c9316c58e52e4a1ce4badfc3bc71e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 18 May 2017 17:47:56 +0100 Subject: [PATCH 01/49] Make a start on kernel discovery framework --- jupyter_client/discovery.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 jupyter_client/discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py new file mode 100644 index 000000000..7e640c32f --- /dev/null +++ b/jupyter_client/discovery.py @@ -0,0 +1,80 @@ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + + +class KernelSpecFinder(object): + """Find kernels from installed kernelspec directories. + """ + id = 'spec' + + def __init__(self): + self.ksm = KernelSpecManager() + + def find_kernels(self): + for name, resdir in self.ksm.find_kernel_specs().items(): + spec = self.ksm._get_kernel_spec_by_name(name, resdir) + yield name, { + # TODO: get full language info + 'language': {'name': spec.language}, + 'display_name': spec.display_name, + 'argv': spec.argv, + } + + def make_manager(self, name): + spec = self.ksm.get_kernel_spec(name) + return KernelManager(kernel_cmd=spec.argv) # TODO: env + + +class IPykernelFinder(object): + """Find ipykernel on this Python version by trying to import it. + """ + id = 'pyimport' + + def _check_for_kernel(self): + try: + from ipykernel.kernelspec import RESOURCES, get_kernel_dict + from ipykernel.ipkernel import IPythonKernel + except ImportError: + return None + else: + return { + 'spec': get_kernel_dict(), + 'language_info': IPythonKernel.language_info, + 'resources_dir': RESOURCES, + } + + def find_kernels(self): + info = self._check_for_kernel() + + if info: + yield 'kernel', { + 'language': info['language_info'], + 'display_name': info['spec']['display_name'], + 'argv': info['spec']['argv'], + } + + def make_manager(self): + info = self._check_for_kernel() + if info is None: + raise Exception("ipykernel is not importable") + return KernelManager(kernel_cmd=info['spec']['argv']) + + +class MetaKernelFinder(object): + def __init__(self): + self.finders = [ + KernelSpecFinder(), + IPykernelFinder(), + ] + + def find_kernels(self): + for finder in self.finders: + for kid, attributes in finder.find_kernels(): + id = finder.id + '/' + kid + yield id, attributes + + def make_manager(self, id): + finder_id, kernel_id = id.split('/', 1) + for finder in self.finders: + if finder_id == finder.id: + return finder.make_manager(kernel_id) From 6406393ff0fcb66fc547f77f81541af1e507978b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 14:31:37 +0100 Subject: [PATCH 02/49] Undeprecate KernelManager.kernel_cmd, add extra_env --- jupyter_client/discovery.py | 4 ++-- jupyter_client/manager.py | 24 ++++++++---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 7e640c32f..737ee38b1 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -22,7 +22,7 @@ def find_kernels(self): def make_manager(self, name): spec = self.ksm.get_kernel_spec(name) - return KernelManager(kernel_cmd=spec.argv) # TODO: env + return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) class IPykernelFinder(object): @@ -53,7 +53,7 @@ def find_kernels(self): 'argv': info['spec']['argv'], } - def make_manager(self): + def make_manager(self, name): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d50a5fbb8..4e0387762 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -22,7 +22,7 @@ from ipython_genutils.importstring import import_item from .localinterfaces import is_local_ip, local_ips from traitlets import ( - Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName + Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, Dict ) from jupyter_client import ( launch_kernel, @@ -87,23 +87,13 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec - kernel_cmd = List(Unicode(), config=True, - help="""DEPRECATED: Use kernel_name instead. - - The Popen Command to launch the kernel. - Override this if you have a custom kernel. - If kernel_cmd is specified in a configuration file, - Jupyter does not pass any arguments to the kernel, - because it cannot make any assumptions about the - arguments that the kernel understands. In particular, - this means that the kernel does not receive the - option --debug if it given on the Jupyter command line. - """ + kernel_cmd = List(Unicode(), + help="""The Popen Command to launch the kernel.""" ) - def _kernel_cmd_changed(self, name, old, new): - warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " - "start different kernels.") + extra_env = Dict( + help="""Extra environment variables to be set for the kernel.""" + ) @property def ipykernel(self): @@ -254,6 +244,8 @@ def start_kernel(self, **kw): # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) + elif self.extra_env: + env.update(self.extra_env) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) From 6ca3ec77ab675a0c1b31dbd3f67538589f1d63cc Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:35:52 +0100 Subject: [PATCH 03/49] Use entry points to find kernel finders --- jupyter_client/discovery.py | 24 +++++++++++++++++++----- setup.py | 4 ++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 737ee38b1..d6ff84b68 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,6 +1,10 @@ +import entrypoints +import logging + from .kernelspec import KernelSpecManager from .manager import KernelManager +log = logging.getLogger(__name__) class KernelSpecFinder(object): """Find kernels from installed kernelspec directories. @@ -61,11 +65,21 @@ def make_manager(self, name): class MetaKernelFinder(object): - def __init__(self): - self.finders = [ - KernelSpecFinder(), - IPykernelFinder(), - ] + def __init__(self, finders): + self.finders = finders + + @classmethod + def from_entrypoints(cls): + finders = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + try: + finder = ep.load()() # Load and instantiate + except Exception: + log.error('Error loading kernel finder', exc_info=True) + else: + finders.append(finder) + + return cls(finders) def find_kernels(self): for finder in self.finders: diff --git a/setup.py b/setup.py index 341af7fb2..a48d4c428 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,10 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + ], + 'jupyter_client.kernel_finders' : [ + 'spec = jupyter_client.discovery:KernelSpecFinder', + 'pyimport = jupyter_client.discovery:IPykernelFinder', ] }, ) From dddda322e93807b9b09af0d8bbf1967c4faa90b8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:56:34 +0100 Subject: [PATCH 04/49] Tests for kernel discovery machinery --- jupyter_client/discovery.py | 21 +++++++++++++++-- jupyter_client/tests/test_discovery.py | 32 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 jupyter_client/tests/test_discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index d6ff84b68..f43cb48a4 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import entrypoints import logging @@ -6,7 +7,23 @@ log = logging.getLogger(__name__) -class KernelSpecFinder(object): +class KernelFinderBase(ABC): + id = None # Should be a short string identifying the finder class. + + @abstractmethod + def find_kernels(self): + """Return an iterator of (kernel_name, kernel_info_dict) tuples.""" + pass + + @abstractmethod + def make_manager(self, name): + """Make and return a KernelManager instance to start a specified kernel + + name will be one of the kernel names produced by find_kernels() + """ + pass + +class KernelSpecFinder(KernelFinderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -29,7 +46,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(object): +class IPykernelFinder(KernelFinderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py new file mode 100644 index 000000000..f6a462327 --- /dev/null +++ b/jupyter_client/tests/test_discovery.py @@ -0,0 +1,32 @@ +import sys + +from jupyter_client import KernelManager +from jupyter_client import discovery + +def test_ipykernel_finder(): + import ipykernel # Fail clearly if ipykernel not installed + ikf = discovery.IPykernelFinder() + + res = list(ikf.find_kernels()) + assert len(res) == 1, res + id, info = res[0] + assert id == 'kernel' + assert info['argv'][0] == sys.executable + +class DummyKernelFinder(discovery.KernelFinderBase): + """A dummy kernel finder for testing MetaKernelFinder""" + id = 'dummy' + + def find_kernels(self): + yield 'sample', {'argv': ['dummy_kernel']} + + def make_manager(self, name): + return KernelManager(kernel_cmd=['dummy_kernel']) + +def test_meta_kernel_finder(): + mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) + assert list(mkf.find_kernels()) == \ + [('dummy/sample', {'argv': ['dummy_kernel']})] + + manager = mkf.make_manager('dummy/sample') + assert manager.kernel_cmd == ['dummy_kernel'] From 1509dacd4c526eb3976577ee14c557abc77c97ed Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 17:47:10 +0100 Subject: [PATCH 05/49] Use older ABC definition style with metaclass --- jupyter_client/discovery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index f43cb48a4..9fd4e6327 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,13 +1,14 @@ -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod import entrypoints import logging +import six from .kernelspec import KernelSpecManager from .manager import KernelManager log = logging.getLogger(__name__) -class KernelFinderBase(ABC): +class KernelFinderBase(six.with_metaclass(ABCMeta, object)): id = None # Should be a short string identifying the finder class. @abstractmethod From 38ccbdc0751a35d46abca80ae64a106d492841cd Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:21:28 +0100 Subject: [PATCH 06/49] Rename kernel finders -> kernel providers MetaKernelFinder -> KernelFinder --- jupyter_client/discovery.py | 55 ++++++++++++++++---------- jupyter_client/tests/test_discovery.py | 14 +++---- setup.py | 2 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 9fd4e6327..53ccab11b 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -8,8 +8,8 @@ log = logging.getLogger(__name__) -class KernelFinderBase(six.with_metaclass(ABCMeta, object)): - id = None # Should be a short string identifying the finder class. +class KernelProviderBase(six.with_metaclass(ABCMeta, object)): + id = None # Should be a short string identifying the provider class. @abstractmethod def find_kernels(self): @@ -24,7 +24,7 @@ def make_manager(self, name): """ pass -class KernelSpecFinder(KernelFinderBase): +class KernelSpecProvider(KernelProviderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -47,7 +47,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(KernelFinderBase): +class IPykernelProvider(KernelProviderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' @@ -82,31 +82,46 @@ def make_manager(self, name): return KernelManager(kernel_cmd=info['spec']['argv']) -class MetaKernelFinder(object): - def __init__(self, finders): - self.finders = finders +class KernelFinder(object): + """Manages a collection of kernel providers to find available kernels + """ + def __init__(self, providers): + self.providers = providers @classmethod def from_entrypoints(cls): - finders = [] - for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + """Load all kernel providers advertised by entry points. + + Kernel providers should use the "jupyter_client.kernel_providers" + entry point group. + + Returns an instance of KernelFinder. + """ + providers = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_providers'): try: - finder = ep.load()() # Load and instantiate + provider = ep.load()() # Load and instantiate except Exception: - log.error('Error loading kernel finder', exc_info=True) + log.error('Error loading kernel provider', exc_info=True) else: - finders.append(finder) + providers.append(provider) - return cls(finders) + return cls(providers) def find_kernels(self): - for finder in self.finders: - for kid, attributes in finder.find_kernels(): - id = finder.id + '/' + kid + """Iterate over available kernels. + + Yields 2-tuples of (id_str, attributes) + """ + for provider in self.providers: + for kid, attributes in provider.find_kernels(): + id = provider.id + '/' + kid yield id, attributes def make_manager(self, id): - finder_id, kernel_id = id.split('/', 1) - for finder in self.finders: - if finder_id == finder.id: - return finder.make_manager(kernel_id) + """Make a KernelManager instance for a given kernel ID. + """ + provider_id, kernel_id = id.split('/', 1) + for provider in self.providers: + if provider_id == provider.id: + return provider.make_manager(kernel_id) diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index f6a462327..9d7833ba3 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -3,9 +3,9 @@ from jupyter_client import KernelManager from jupyter_client import discovery -def test_ipykernel_finder(): +def test_ipykernel_provider(): import ipykernel # Fail clearly if ipykernel not installed - ikf = discovery.IPykernelFinder() + ikf = discovery.IPykernelProvider() res = list(ikf.find_kernels()) assert len(res) == 1, res @@ -13,8 +13,8 @@ def test_ipykernel_finder(): assert id == 'kernel' assert info['argv'][0] == sys.executable -class DummyKernelFinder(discovery.KernelFinderBase): - """A dummy kernel finder for testing MetaKernelFinder""" +class DummyKernelProvider(discovery.KernelProviderBase): + """A dummy kernel provider for testing KernelFinder""" id = 'dummy' def find_kernels(self): @@ -24,9 +24,9 @@ def make_manager(self, name): return KernelManager(kernel_cmd=['dummy_kernel']) def test_meta_kernel_finder(): - mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) - assert list(mkf.find_kernels()) == \ + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + assert list(kf.find_kernels()) == \ [('dummy/sample', {'argv': ['dummy_kernel']})] - manager = mkf.make_manager('dummy/sample') + manager = kf.make_manager('dummy/sample') assert manager.kernel_cmd == ['dummy_kernel'] diff --git a/setup.py b/setup.py index a48d4c428..2399a8139 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def run(self): 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], - 'jupyter_client.kernel_finders' : [ + 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecFinder', 'pyimport = jupyter_client.discovery:IPykernelFinder', ] From 3c09a5732d569be86fd92f06b3deeab6413c4e65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:26:52 +0100 Subject: [PATCH 07/49] Missed a rename --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2399a8139..099ddc90a 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,8 @@ def run(self): 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], 'jupyter_client.kernel_providers' : [ - 'spec = jupyter_client.discovery:KernelSpecFinder', - 'pyimport = jupyter_client.discovery:IPykernelFinder', + 'spec = jupyter_client.discovery:KernelSpecProvider', + 'pyimport = jupyter_client.discovery:IPykernelProvider', ] }, ) From e92e5c194adde5188039269255a0f82c2c45627d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:41:43 +0100 Subject: [PATCH 08/49] Add dependency on entrypoints --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 099ddc90a..f042f00b3 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ def run(self): 'jupyter_core', 'pyzmq>=13', 'python-dateutil>=2.1', + 'entrypoints', ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], From aad40cb6ebb785f8ce51ee70b9872d62aef09e32 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:17:58 +0100 Subject: [PATCH 09/49] Document new kernel providers system --- docs/index.rst | 1 + docs/kernel_providers.rst | 146 ++++++++++++++++++++++++++++++++++++ jupyter_client/discovery.py | 18 +++-- 3 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 docs/kernel_providers.rst diff --git a/docs/index.rst b/docs/index.rst index a0b8855cc..41e218ccc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ with Jupyter kernels. kernels wrapperkernels + kernel_providers .. toctree:: :maxdepth: 2 diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst new file mode 100644 index 000000000..8d51ec7f4 --- /dev/null +++ b/docs/kernel_providers.rst @@ -0,0 +1,146 @@ +================ +Kernel providers +================ + +.. note:: + This is a new interface under development. Not all Jupyter applications + use this yet. See :ref:`kernelspecs` for the established way of discovering + kernel types. + +By writing a kernel provider, you can extend how Jupyter applications discover +and start kernels. To do so, subclass +:class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID +and overriding two methods. + +.. class:: MyKernelProvider + + .. attribute:: id + + A short string identifying this provider. Cannot contain forward slash + (``/``). + + .. method:: find_kernels() + + Get the available kernel types this provider knows about. + Return an iterable of 2-tuples: (name, attributes). + *name* is a short string identifying the kernel type. + *attributes* is a dictionary with information to allow selecting a kernel. + + .. method:: make_manager(name) + + Prepare and return a :class:`~jupyter_client.KernelManager` instance + ready to start a new kernel instance of the type identified by *name*. + The input will be one of the names given by :meth:`find_kernels`. + +For example, imagine we want to tell Jupyter about kernels for a new language +called *oblong*:: + + # oblong_provider.py + from jupyter_client.discover import KernelProviderBase + from jupyter_client import KernelManager + from shutil import which + + class OblongKernelProvider(KernelProviderBase): + id = 'oblong' + + def find_kernels(self): + if not which('oblong-kernel'): + return # Check it's available + + # Two variants - for a real kernel, these could be different + # environments + yield 'standard', { + 'display_name': 'Oblong (standard)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + yield 'rounded', { + 'display_name': 'Oblong (rounded)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + + def make_manager(self, name): + if name == 'standard': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '0'}) + elif name == 'rounded': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '1'}) + else: + raise ValueError("Unknown kernel %s" % name) + +You would then register this with an *entry point*. In your ``setup.py``, put +something like this:: + + setup(... + entry_points = { + 'jupyter_client.kernel_providers' : [ + # The name before the '=' should match the id attribute + 'oblong = oblong_provider:OblongKernelProvider', + ] + }) + +To find and start kernels in client code, use +:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel +providers, but it wraps a set of kernel providers. The kernel names it works +with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example +above). + +:: + + from jupyter_client.discovery import KernelFinder + kf = KernelFinder.from_entrypoints() + + ## Find available kernel types + for name, attributes in kf.find_kernels(): + print(name, ':', attributes['display_name']) + # oblong/standard : Oblong (standard) + # oblong/rounded : Oblong(rounded) + # ... + + ## Start a kernel by name + manager = kf.make_manager('oblong/standard') + manager.start_kernel() + +.. module:: jupyter_client.discovery + +.. autoclass:: KernelFinder + + .. automethod:: from_entrypoints + + .. automethod:: find_kernels + + .. automethod:: make_manager + +Included kernel providers +========================= + +``jupyter_client`` includes two kernel providers: + +.. autoclass:: KernelSpecProvider + + .. seealso:: :ref:`kernelspecs` + +.. autoclass:: IPykernelProvider + +Glossary +======== + +Kernel instance + A running kernel, a process which can accept ZMQ connections from frontends. + Its state includes a namespace and an execution counter. + +Kernel type + Allows starting multiple, initially similar kernel instances. The kernel type + entails the combination of software to run the kernel, and the context in + which it starts. For instance, one kernel type may be associated with one + conda environment containing ``ipykernel``. The same kernel software in + another environment would be a different kernel type. Another software package + for a kernel, such as ``IRkernel``, would also be a different kernel type. + +Kernel provider + A Python class to discover kernel types and allow a client to start instances + of those kernel types. For instance, one kernel provider might find conda + environments containing ``ipykernel`` and allow starting kernel instances in + these environments. diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 53ccab11b..6f6b52f0d 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -25,7 +25,7 @@ def make_manager(self, name): pass class KernelSpecProvider(KernelProviderBase): - """Find kernels from installed kernelspec directories. + """Offers kernel types from installed kernelspec directories. """ id = 'spec' @@ -48,7 +48,9 @@ def make_manager(self, name): class IPykernelProvider(KernelProviderBase): - """Find ipykernel on this Python version by trying to import it. + """Offers a kernel type using the Python interpreter it's running in. + + This checks if ipykernel is importable first. """ id = 'pyimport' @@ -83,7 +85,9 @@ def make_manager(self, name): class KernelFinder(object): - """Manages a collection of kernel providers to find available kernels + """Manages a collection of kernel providers to find available kernel types + + *providers* should be a list of kernel provider instances. """ def __init__(self, providers): self.providers = providers @@ -109,17 +113,17 @@ def from_entrypoints(cls): return cls(providers) def find_kernels(self): - """Iterate over available kernels. + """Iterate over available kernel types. - Yields 2-tuples of (id_str, attributes) + Yields 2-tuples of (prefixed_name, attributes) """ for provider in self.providers: for kid, attributes in provider.find_kernels(): id = provider.id + '/' + kid yield id, attributes - def make_manager(self, id): - """Make a KernelManager instance for a given kernel ID. + def make_manager(self, name): + """Make a KernelManager instance for a given kernel type. """ provider_id, kernel_id = id.split('/', 1) for provider in self.providers: From c09b8aced5912901b4e96e11d9d4eeb3346b4fd7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:25:41 +0100 Subject: [PATCH 10/49] Break it up a bit with a subheading --- docs/kernel_providers.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 8d51ec7f4..c5a62cc3d 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -81,6 +81,9 @@ something like this:: ] }) +Finding kernel types +==================== + To find and start kernels in client code, use :class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel providers, but it wraps a set of kernel providers. The kernel names it works From cc8176b088fdf7a5ea955fec980c6b90e8bd21f7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:06:43 +0100 Subject: [PATCH 11/49] Update doc with Carol's suggestions --- docs/kernel_providers.rst | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index c5a62cc3d..65cbd9c8f 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -3,12 +3,18 @@ Kernel providers ================ .. note:: - This is a new interface under development. Not all Jupyter applications - use this yet. See :ref:`kernelspecs` for the established way of discovering - kernel types. + This is a new interface under development, and may still change. + Not all Jupyter applications use this yet. + See :ref:`kernelspecs` for the established way of discovering kernel types. + +Creating a kernel provider +========================== By writing a kernel provider, you can extend how Jupyter applications discover -and start kernels. To do so, subclass +and start kernels. For example, you could find kernels in an environment system +like conda, or kernels on remote systems which you can access. + +To write a kernel provider, subclass :class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID and overriding two methods. @@ -47,8 +53,8 @@ called *oblong*:: if not which('oblong-kernel'): return # Check it's available - # Two variants - for a real kernel, these could be different - # environments + # Two variants - for a real kernel, these could be something like + # different conda environments. yield 'standard', { 'display_name': 'Oblong (standard)', 'language': {'name': 'oblong'}, @@ -85,8 +91,9 @@ Finding kernel types ==================== To find and start kernels in client code, use -:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel -providers, but it wraps a set of kernel providers. The kernel names it works +:class:`jupyter_client.discovery.KernelFinder`. This uses multiple kernel +providers to find available kernels. Like a kernel provider, it has methods +``find_kernels`` and ``make_manager``. The kernel names it works with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example above). @@ -116,8 +123,8 @@ above). .. automethod:: make_manager -Included kernel providers -========================= +Kernel providers included in ``jupyter_client`` +=============================================== ``jupyter_client`` includes two kernel providers: @@ -135,9 +142,9 @@ Kernel instance Its state includes a namespace and an execution counter. Kernel type - Allows starting multiple, initially similar kernel instances. The kernel type - entails the combination of software to run the kernel, and the context in - which it starts. For instance, one kernel type may be associated with one + The software to run a kernel instance, along with the context in which a + kernel starts. One kernel type allows starting multiple, initially similar + kernel instances. For instance, one kernel type may be associated with one conda environment containing ``ipykernel``. The same kernel software in another environment would be a different kernel type. Another software package for a kernel, such as ``IRkernel``, would also be a different kernel type. From 1f74c5f40f9948b1978451ceb473beb319cd7649 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:07:53 +0100 Subject: [PATCH 12/49] Fix variable name --- jupyter_client/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 6f6b52f0d..2bfe92b2a 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -125,7 +125,7 @@ def find_kernels(self): def make_manager(self, name): """Make a KernelManager instance for a given kernel type. """ - provider_id, kernel_id = id.split('/', 1) + provider_id, kernel_id = name.split('/', 1) for provider in self.providers: if provider_id == provider.id: return provider.make_manager(kernel_id) From 16608fc7835cba0bbd9e62feac31481dcd517c64 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Wed, 18 Oct 2017 20:29:25 -0400 Subject: [PATCH 13/49] Fix typo in documentation. --- docs/kernel_providers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 65cbd9c8f..2e1b7e295 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -42,7 +42,7 @@ For example, imagine we want to tell Jupyter about kernels for a new language called *oblong*:: # oblong_provider.py - from jupyter_client.discover import KernelProviderBase + from jupyter_client.discovery import KernelProviderBase from jupyter_client import KernelManager from shutil import which From 936dfe0584441ababc8e6d86740f4791f7739a19 Mon Sep 17 00:00:00 2001 From: frelon Date: Wed, 1 Nov 2017 12:56:55 +0100 Subject: [PATCH 14/49] Updated URL for Jupyter Kernels The old URL points to a "This page has moved"-page --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 3319dda31..2fe1500aa 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -6,7 +6,7 @@ Making kernels for Jupyter A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for -`several other languages `_. +`several other languages `_. When Jupyter starts a kernel, it passes it a connection file. This specifies how to set up communications with the frontend. From aca5f7084014ec69d51f5141a4fd1bdfb1aa3a3b Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 10 Nov 2017 14:43:08 +0100 Subject: [PATCH 15/49] tornado 5 support - use IOLoop.current over IOLoop.instance - drop removed `loop` arg from PeriodicCallback - deprecate now-unused IOLoopKernelRestarter.loop --- jupyter_client/ioloop/manager.py | 20 ++++---------------- jupyter_client/ioloop/restarter.py | 27 +++++++++------------------ jupyter_client/session.py | 4 ++-- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/jupyter_client/ioloop/manager.py b/jupyter_client/ioloop/manager.py index 511a73f55..cc285291b 100644 --- a/jupyter_client/ioloop/manager.py +++ b/jupyter_client/ioloop/manager.py @@ -1,15 +1,7 @@ """A kernel manager with a tornado IOLoop""" -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import @@ -24,10 +16,6 @@ from jupyter_client.manager import KernelManager from .restarter import IOLoopKernelRestarter -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - def as_zmqstream(f): def wrapped(self, *args, **kwargs): @@ -37,9 +25,9 @@ def wrapped(self, *args, **kwargs): class IOLoopKernelManager(KernelManager): - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + return ioloop.IOLoop.current() restarter_class = Type( default_value=IOLoopKernelRestarter, diff --git a/jupyter_client/ioloop/restarter.py b/jupyter_client/ioloop/restarter.py index 6f531744c..69079eecf 100644 --- a/jupyter_client/ioloop/restarter.py +++ b/jupyter_client/ioloop/restarter.py @@ -4,37 +4,28 @@ restarts the kernel if it dies. """ -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import +import warnings from zmq.eventloop import ioloop - from jupyter_client.restarter import KernelRestarter from traitlets import ( Instance, ) -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - class IOLoopKernelRestarter(KernelRestarter): """Monitor and autorestart a kernel.""" - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + warnings.warn("IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2", + DeprecationWarning, stacklevel=4, + ) + return ioloop.IOLoop.current() _pcallback = None @@ -42,7 +33,7 @@ def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( - self.poll, 1000*self.time_to_dead, self.loop + self.poll, 1000*self.time_to_dead, ) self._pcallback.start() diff --git a/jupyter_client/session.py b/jupyter_client/session.py index af60ac259..33b1c0b4a 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -191,9 +191,9 @@ def _context_default(self): session = Instance('jupyter_client.session.Session', allow_none=True) - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return IOLoop.instance() + return IOLoop.current() def __init__(self, **kwargs): super(SessionFactory, self).__init__(**kwargs) From 172d6cdea80bf189a894171fdd39cc6031ae562d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 21 Sep 2017 15:04:12 +0200 Subject: [PATCH 16/49] Configure interrupt mode via spec. - interrupt_mode="signal" is the default and current behaviour - With interrupt_mode="message", instead of a signal, a `interrupt_request` message on the control port will be sent --- jupyter_client/kernelspec.py | 18 ++++++++++++------ jupyter_client/manager.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3465ac7a4..d2248cc58 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -13,7 +13,9 @@ pjoin = os.path.join from ipython_genutils.py3compat import PY3 -from traitlets import HasTraits, List, Unicode, Dict, Set, Bool, Type +from traitlets import ( + HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum +) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH @@ -28,6 +30,9 @@ class KernelSpec(HasTraits): language = Unicode() env = Dict() resource_dir = Unicode() + interrupt_mode = CaselessStrEnum( + ['message', 'signal'], default_value='signal' + ) metadata = Dict() @classmethod @@ -46,6 +51,7 @@ def to_dict(self): env=self.env, display_name=self.display_name, language=self.language, + interrupt_mode=self.interrupt_mode, metadata=self.metadata, ) @@ -227,7 +233,7 @@ def get_all_specs(self): def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. - + Returns the path that was deleted. """ save_native = self.ensure_native_kernel @@ -263,7 +269,7 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, If ``user`` is False, it will attempt to install into the systemwide kernel registry. If the process does not have appropriate permissions, an :exc:`OSError` will be raised. - + If ``prefix`` is given, the kernelspec will be installed to PREFIX/share/jupyter/kernels/KERNEL_NAME. This can be sys.prefix for installation inside virtual or conda envs. @@ -284,16 +290,16 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, DeprecationWarning, stacklevel=2, ) - + destination = self._get_destination_dir(kernel_name, user=user, prefix=prefix) self.log.debug('Installing kernelspec in %s', destination) - + kernel_dir = os.path.dirname(destination) if kernel_dir not in self.kernel_dirs: self.log.warning("Installing to %s, which is not in %s. The kernelspec may not be found.", kernel_dir, self.kernel_dirs, ) - + if os.path.isdir(destination): self.log.info('Removing existing kernelspec in %s', destination) shutil.rmtree(destination) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 4e0387762..2bcc1629a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -246,7 +246,7 @@ def start_kernel(self, **kw): env.update(self.kernel_spec.env or {}) elif self.extra_env: env.update(self.extra_env) - + # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, @@ -403,11 +403,18 @@ def interrupt_kernel(self): platforms. """ if self.has_kernel: - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - self.signal_kernel(signal.SIGINT) + interrupt_mode = self.kernel_spec.interrupt_mode + if interrupt_mode == 'signal': + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) + else: + self.signal_kernel(signal.SIGINT) + + elif interrupt_mode == 'message': + msg = self.session.msg("interrupt_request", content={}) + self._connect_control_socket() + self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") From f0e33ba7532ab50ffd54fbc0912edc61815eba03 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 30 Oct 2017 16:30:12 +0100 Subject: [PATCH 17/49] Update docs. --- docs/kernels.rst | 7 +++++++ docs/messaging.rst | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/kernels.rst b/docs/kernels.rst index 2fe1500aa..76fa67699 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -132,6 +132,13 @@ JSON serialised dictionary containing the following keys and values: is found, a kernel with a matching `language` will be used. This allows a notebook written on any Python or Julia kernel to be properly associated with the user's Python or Julia kernel, even if they aren't listed under the same name as the author's. +- **interrupt_mode** (optional): May be either ``signal`` or ``message`` and + specifies how a client is supposed to interrupt cell execution on this kernel, + either by sending an interrupt ``signal`` via the operating system's + signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + ``interrupt_request`` message on the control channel (see + :ref:`msging_interrupt`). If this is not specified + the client will default to ``signal`` mode. - **env** (optional): A dictionary of environment variables to set for the kernel. These will be added to the current environment variables before the kernel is started. diff --git a/docs/messaging.rst b/docs/messaging.rst index 776dda681..ec8efd99f 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -959,6 +959,27 @@ Message type: ``shutdown_reply``:: socket, they simply send a forceful process termination signal, since a dead process is unlikely to respond in any useful way to messages. +.. _msging_interrupt: + +Kernel interrupt +---------------- + +In case a kernel can not catch operating system interrupt signals (e.g. the used +runtime handles signals and does not allow a user program to define a callback), +a kernel can choose to be notified using a message instead. For this to work, +the kernels kernelspec must set `interrupt_mode` to ``message``. An interruption +will then result in the following message on the `control` channel: + +Message type: ``interrupt_request``:: + + content = {} + +Message type: ``interrupt_reply``:: + + content = {} + +.. versionadded:: 5.3 + Messages on the IOPub (PUB/SUB) channel ======================================= From 21b95699dcb5917a8cf87c8aae3bd67b9e281f3c Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 6 Nov 2017 11:59:10 +0100 Subject: [PATCH 18/49] Bump protocol version. --- docs/messaging.rst | 2 +- jupyter_client/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/messaging.rst b/docs/messaging.rst index ec8efd99f..7c533a7de 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -21,7 +21,7 @@ Versioning The Jupyter message specification is versioned independently of the packages that use it. -The current version of the specification is 5.2. +The current version of the specification is 5.3. .. note:: *New in* and *Changed in* messages in this document refer to versions of the diff --git a/jupyter_client/_version.py b/jupyter_client/_version.py index 90dd2e93e..7f96345ae 100644 --- a/jupyter_client/_version.py +++ b/jupyter_client/_version.py @@ -1,5 +1,5 @@ version_info = (5, 1, 0) __version__ = '.'.join(map(str, version_info)) -protocol_version_info = (5, 2) +protocol_version_info = (5, 3) protocol_version = "%i.%i" % protocol_version_info From 6674afae21cce681c1fae6a37879d40c181cc91c Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 13 Nov 2017 14:31:20 +0100 Subject: [PATCH 19/49] disable pyzmq zero-copy optimizations during session tests --- jupyter_client/tests/test_session.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 43819a898..e80274367 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -8,6 +8,10 @@ import sys import uuid from datetime import datetime +try: + from unittest import mock +except ImportError: + import mock import pytest @@ -34,6 +38,14 @@ def setUp(self): self.session = ss.Session() +@pytest.fixture +def no_copy_threshold(): + """Disable zero-copy optimizations in pyzmq >= 17""" + with mock.patch.object(zmq, 'COPY_THRESHOLD', 1): + yield + + +@pytest.mark.usefixtures('no_copy_threshold') class TestSession(SessionTestCase): def test_msg(self): From e2772bd54c864b805b1cac36b3141fe27b1ba726 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 13 Nov 2017 15:11:37 +0100 Subject: [PATCH 20/49] Fix signal name. --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 76fa67699..5308c603f 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -135,7 +135,7 @@ JSON serialised dictionary containing the following keys and values: - **interrupt_mode** (optional): May be either ``signal`` or ``message`` and specifies how a client is supposed to interrupt cell execution on this kernel, either by sending an interrupt ``signal`` via the operating system's - signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + signalling facilities (e.g. `SIGINT` on POSIX systems), or by sending an ``interrupt_request`` message on the control channel (see :ref:`msging_interrupt`). If this is not specified the client will default to ``signal`` mode. From 250178fe53dcf5c20098e29ea94951caa3aa371e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 10 Feb 2017 16:57:22 +0000 Subject: [PATCH 21/49] Add 'jupyter kernel' command A simple lead in to the 'kernel nanny' work, this adds a command so you can do: jupyter kernel --kernel python --- jupyter_client/kernelapp.py | 66 +++++++++++++++++++++++++++++++++++++ scripts/jupyter-kernel | 5 +++ setup.py | 1 + 3 files changed, 72 insertions(+) create mode 100644 jupyter_client/kernelapp.py create mode 100755 scripts/jupyter-kernel diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py new file mode 100644 index 000000000..4c1c99e3c --- /dev/null +++ b/jupyter_client/kernelapp.py @@ -0,0 +1,66 @@ +import os +import signal +import uuid + +from jupyter_core.application import JupyterApp +from tornado.ioloop import IOLoop +from traitlets import Unicode + +from . import __version__ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + +class KernelApp(JupyterApp): + version = __version__ + description = "Run a kernel locally" + + classes = [KernelManager, KernelSpecManager] + + aliases = { + 'kernel': 'KernelApp.kernel_name', + 'ip': 'KernelManager.ip', + } + + kernel_name = Unicode( + help = 'The name of a kernel to start' + ).tag(config=True) + + def initialize(self, argv=None): + super(KernelApp, self).initialize(argv) + self.km = KernelManager(kernel_name=self.kernel_name, + config=self.config) + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() + + def setup_signals(self): + if os.name == 'nt': + return + + def shutdown_handler(signo, frame): + self.loop.add_callback_from_signal(self.shutdown, signo) + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, shutdown_handler) + + def shutdown(self, signo): + self.log.info('Shutting down on signal %d' % signo) + self.km.shutdown_kernel() + self.loop.stop() + + def log_connection_info(self): + cf = self.km.connection_file + self.log.info('Connection file: %s', cf) + self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + + def start(self): + self.log.info('Starting kernel %r', self.kernel_name) + try: + self.km.start_kernel() + self.log_connection_info() + self.setup_signals() + self.loop.start() + finally: + self.km.cleanup() + + +main = KernelApp.launch_instance diff --git a/scripts/jupyter-kernel b/scripts/jupyter-kernel new file mode 100755 index 000000000..31144d405 --- /dev/null +++ b/scripts/jupyter-kernel @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from jupyter_client.kernelapp import main + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index f042f00b3..022cbc56e 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecProvider', From 9359b338c90f8e259ac4b307d99ce20ca3b2cbf7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:23:18 +0000 Subject: [PATCH 22/49] Use native kernel by default --- jupyter_client/kernelapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 4c1c99e3c..071a0f3ed 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -7,7 +7,7 @@ from traitlets import Unicode from . import __version__ -from .kernelspec import KernelSpecManager +from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME from .manager import KernelManager class KernelApp(JupyterApp): @@ -21,7 +21,7 @@ class KernelApp(JupyterApp): 'ip': 'KernelManager.ip', } - kernel_name = Unicode( + kernel_name = Unicode(NATIVE_KERNEL_NAME, help = 'The name of a kernel to start' ).tag(config=True) From ae03ddde10c215a8df1efe4a29d5bfa91b1efdfa Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:32:40 +0000 Subject: [PATCH 23/49] More description --- jupyter_client/kernelapp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 071a0f3ed..799d85ee4 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -2,7 +2,7 @@ import signal import uuid -from jupyter_core.application import JupyterApp +from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode @@ -11,8 +11,10 @@ from .manager import KernelManager class KernelApp(JupyterApp): + """Launch a kernel by name in a local subprocess. + """ version = __version__ - description = "Run a kernel locally" + description = "Run a kernel locally in a subprocess" classes = [KernelManager, KernelSpecManager] @@ -20,9 +22,10 @@ class KernelApp(JupyterApp): 'kernel': 'KernelApp.kernel_name', 'ip': 'KernelManager.ip', } + flags = {'debug': base_flags['debug']} kernel_name = Unicode(NATIVE_KERNEL_NAME, - help = 'The name of a kernel to start' + help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): @@ -34,6 +37,7 @@ def initialize(self, argv=None): self.loop = IOLoop.current() def setup_signals(self): + """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" if os.name == 'nt': return From 7e6d16711c6f16782a497dc6dbf76911c334f46e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:22:38 +0000 Subject: [PATCH 24/49] Add test of 'jupyter kernel' --- jupyter_client/kernelapp.py | 11 +++++ jupyter_client/tests/test_kernelapp.py | 57 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 jupyter_client/tests/test_kernelapp.py diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 799d85ee4..a2ab17812 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -35,6 +35,7 @@ def initialize(self, argv=None): cf_basename = 'kernel-%s.json' % uuid.uuid4() self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) self.loop = IOLoop.current() + self.loop.add_callback(self._record_started) def setup_signals(self): """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" @@ -56,6 +57,16 @@ def log_connection_info(self): self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + def _record_started(self): + """For tests, create a file to indicate that we've started + + Do not rely on this except in our own tests! + """ + fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') + if fn is not None: + with open(fn, 'wb'): + pass + def start(self): self.log.info('Starting kernel %r', self.kernel_name) try: diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py new file mode 100644 index 000000000..b41a02bc6 --- /dev/null +++ b/jupyter_client/tests/test_kernelapp.py @@ -0,0 +1,57 @@ +from __future__ import division + +import os +import shutil +from subprocess import Popen, PIPE +import sys +from tempfile import mkdtemp +import time + +def _launch(extra_env): + env = os.environ.copy() + env.update(extra_env) + return Popen([sys.executable, '-c', + 'from jupyter_client.kernelapp import main; main()'], + env=env, stderr=PIPE) + +WAIT_TIME = 10 +POLL_FREQ = 10 + +def test_kernelapp_lifecycle(): + # Check that 'jupyter kernel' starts and terminates OK. + runtime_dir = mkdtemp() + startup_dir = mkdtemp() + started = os.path.join(startup_dir, 'started') + try: + p = _launch({'JUPYTER_RUNTIME_DIR': runtime_dir, + 'JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE': started, + }) + # Wait for start + for _ in range(WAIT_TIME * POLL_FREQ): + if os.path.isfile(started): + break + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("No started file created in {} seconds" + .format(WAIT_TIME)) + + # Connection file should be there by now + files = os.listdir(runtime_dir) + assert len(files) == 1 + cf = files[0] + assert cf.startswith('kernel') + assert cf.endswith('.json') + + # Read the first three lines from stderr. This will hang if there are + # fewer lines to read; I don't see any way to avoid that without lots + # of extra complexity. + b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') + assert cf in b + + # Send SIGTERM to shut down + p.terminate() + p.wait(timeout=10) + finally: + shutil.rmtree(runtime_dir) + shutil.rmtree(startup_dir) + From 28f908f0da34ed0e0c85f58016334c434c18bb5f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:32:35 +0000 Subject: [PATCH 25/49] Workaround lack of timeout on Py2 --- jupyter_client/tests/test_kernelapp.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py index b41a02bc6..2533472d4 100644 --- a/jupyter_client/tests/test_kernelapp.py +++ b/jupyter_client/tests/test_kernelapp.py @@ -7,16 +7,28 @@ from tempfile import mkdtemp import time +PY3 = sys.version_info[0] >= 3 + def _launch(extra_env): env = os.environ.copy() env.update(extra_env) return Popen([sys.executable, '-c', 'from jupyter_client.kernelapp import main; main()'], - env=env, stderr=PIPE) + env=env, stderr=(PIPE if PY3 else None)) WAIT_TIME = 10 POLL_FREQ = 10 +def hacky_wait(p): + """Python 2 subprocess doesn't have timeouts :-(""" + for _ in range(WAIT_TIME * POLL_FREQ): + if p.poll() is not None: + return p.returncode + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("Process didn't exit in {} seconds" + .format(WAIT_TIME)) + def test_kernelapp_lifecycle(): # Check that 'jupyter kernel' starts and terminates OK. runtime_dir = mkdtemp() @@ -42,15 +54,13 @@ def test_kernelapp_lifecycle(): assert cf.startswith('kernel') assert cf.endswith('.json') - # Read the first three lines from stderr. This will hang if there are - # fewer lines to read; I don't see any way to avoid that without lots - # of extra complexity. - b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') - assert cf in b - # Send SIGTERM to shut down p.terminate() - p.wait(timeout=10) + if PY3: + _, stderr = p.communicate(timeout=WAIT_TIME) + assert cf in stderr.decode('utf-8', 'replace') + else: + hacky_wait(p) finally: shutil.rmtree(runtime_dir) shutil.rmtree(startup_dir) From aa8b184c9c8134cae731c4652a87a704b6ee9f65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:34:19 +0000 Subject: [PATCH 26/49] Restrict to older pytest on Python 3.3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 022cbc56e..1230f2142 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test:python_version == "3.3"': ['pytest<3.3.0'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From 5291f940c8cac341ed96c6b2dd73bbdd11db1df5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:36:43 +0000 Subject: [PATCH 27/49] Another go at fixing pytest dependency on Python 3.3 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1230f2142..233f83a0e 100644 --- a/setup.py +++ b/setup.py @@ -85,8 +85,9 @@ def run(self): 'entrypoints', ], extras_require = { - 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], + 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From e952b835f6c9a58c901fca4663a4cf473eac1284 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 29 Nov 2017 15:53:39 +0000 Subject: [PATCH 28/49] First go at new launcher APIs --- jupyter_client/async_launcher.py | 95 ++++++++++ jupyter_client/launcher2.py | 287 +++++++++++++++++++++++++++++++ 2 files changed, 382 insertions(+) create mode 100644 jupyter_client/async_launcher.py create mode 100644 jupyter_client/launcher2.py diff --git a/jupyter_client/async_launcher.py b/jupyter_client/async_launcher.py new file mode 100644 index 000000000..d62069249 --- /dev/null +++ b/jupyter_client/async_launcher.py @@ -0,0 +1,95 @@ +"""Launch and control kernels using asyncio. +""" +from abc import ABC, abstractmethod +import asyncio + +from .launcher2 import make_connection_file, build_popen_kwargs + +class AsyncKernelLauncher(ABC): + """Interface for async kernel launchers. + + This is very similar to the KernelLauncher interface, but its methods + are coroutines. + """ + @abstractmethod + def launch(self): + """Launch the kernel.""" + + @abstractmethod + def wait(self): + """Wait for the kernel process to exit. + """ + raise NotImplementedError() + + @abstractmethod + def send_signal(self, signum): + """Send a signal to the kernel.""" + pass + + def cleanup(self): + """Clean up any resources.""" + pass + + +class AsyncPopenKernelLauncher(AsyncKernelLauncher): + """Launch a kernel asynchronously in a subprocess. + + This is the async counterpart to PopenKernelLauncher. + """ + process = None + connection_file = None + connection_info = None + + def __init__(self, cmd_template, extra_env=None, cwd=None): + self.cmd_template = cmd_template + self.extra_env = extra_env + self.cwd = cwd + + @asyncio.coroutine + def launch(self): + self.connection_file, self.connection_info = make_connection_file() + kwargs = build_popen_kwargs(self.cmd_template, self.connection_file, + self.extra_env, self.cwd) + args = kwargs.pop('args') + self.process = yield from asyncio.create_subprocess_exec(*args, **kwargs) + + @asyncio.coroutine + def wait(self): + return (yield from self.process.wait()) + + @asyncio.coroutine + def send_signal(self, signum): + return self.process.send_signal(signum) + + @asyncio.coroutine + def cleanup(self): + super().cleanup() + +class AsyncLauncherWrapper(AsyncKernelLauncher): + """Wrap a blocking KernelLauncher to be used asynchronously. + + This calls the blocking methods in the event loop's default executor. + """ + def __init__(self, wrapped, loop=None): + self.wrapped = wrapped + self.loop = loop or asyncio.get_event_loop() + + def in_default_executor(self, f, *args): + return self.loop.run_in_executor(None, f, *args) + + @asyncio.coroutine + def launch(self): + return (yield from self.in_default_executor(self.wrapped.launch)) + + @asyncio.coroutine + def wait(self): + return (yield from self.in_default_executor(self.wrapped.wait)) + + @asyncio.coroutine + def send_signal(self, signum): + return (yield from self.in_default_executor(self.wrapped.send_signal, signum)) + + @asyncio.coroutine + def cleanup(self): + return (yield from self.in_default_executor(self.wrapped.cleanup)) + diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py new file mode 100644 index 000000000..0b2d8b6e3 --- /dev/null +++ b/jupyter_client/launcher2.py @@ -0,0 +1,287 @@ +from abc import ABCMeta, abstractmethod +from binascii import b2a_hex +import errno +import json +import os +import re +import six +import socket +import stat +from subprocess import Popen, PIPE +import sys +import warnings + +from ipython_genutils.encoding import getdefaultencoding +from ipython_genutils.py3compat import cast_bytes_py2 +from .localinterfaces import localhost +from jupyter_core.paths import jupyter_runtime_dir + +def new_key(): + """Generate a new random key string. + + Avoids problematic runtime import in stdlib uuid on Python 2. + + Returns + ------- + + id string (16 random bytes as hex-encoded text, chunks separated by '-') + """ + buf = os.urandom(16) + return u'-'.join(b2a_hex(x).decode('ascii') for x in ( + buf[:4], buf[4:] + )) + +def random_ports(ip, transport='tcp'): + """Pick a set of random, unused ports for the kernel to use. + """ + res = {} + port_names = ['shell_port', 'iopub_port', 'stdin_port', 'control_port', + 'hb_port'] + if transport == 'tcp': + # store sockets temporarily to avoid reusing a port number + tmp_socks = [] + for _ in port_names: + sock = socket.socket() + # struct.pack('ii', (0,0)) is 8 null bytes + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) + sock.bind((ip, 0)) + tmp_socks.append(sock) + for name, sock in zip(port_names, tmp_socks): + port = sock.getsockname()[1] + sock.close() + res[name] = port + else: + N = 1 + for name in port_names: + while os.path.exists("%s-%s" % (ip, str(N))): + N += 1 + res[name] = N + N += 1 + return res + +def set_sticky_bit(fname): + """Set the sticky bit on the file and its parent directory. + + This stops it being deleted by periodic cleanup of XDG_RUNTIME_DIR. + """ + if not hasattr(stat, 'S_ISVTX'): + return + + paths = [fname] + runtime_dir = os.path.dirname(fname) + if runtime_dir: + paths.append(runtime_dir) + for path in paths: + permissions = os.stat(path).st_mode + new_permissions = permissions | stat.S_ISVTX + if new_permissions != permissions: + try: + os.chmod(path, new_permissions) + except OSError as e: + if e.errno == errno.EPERM and path == runtime_dir: + # suppress permission errors setting sticky bit on runtime_dir, + # which we may not own. + pass + else: + # failed to set sticky bit, probably not a big deal + warnings.warn( + "Failed to set sticky bit on %r: %s" + "\nProbably not a big deal, but runtime files may be cleaned up periodically." % (path, e), + RuntimeWarning, + ) + +def make_connection_file(ip=None, transport='tcp'): + """Generates a JSON config file, including the selection of random ports. + + Parameters + ---------- + + fname : unicode + The path to the file to write + + ip : str, optional + The ip address the kernel will bind to. + + transport : str, optional + The ZMQ transport to use: tcp or ipc + """ + if not ip: + ip = localhost() + + fname = os.path.join(jupyter_runtime_dir(), 'kernel-%s.json' % new_key()) + + cfg = random_ports(ip=ip, transport=transport) + cfg['ip'] = ip + cfg['key'] = new_key() + cfg['transport'] = transport + cfg['signature_scheme'] = 'hmac-sha256' + + with open(fname, 'w') as f: + f.write(json.dumps(cfg, indent=2)) + + set_sticky_bit(fname) + + return fname, cfg + +def format_kernel_cmd(cmd, connection_file, kernel_resource_dir=None): + """Replace templated args (e.g. {connection_file}) + """ + if cmd and cmd[0] == 'python': + # executable is 'python', use sys.executable. + # These will typically be the same, + # but if the current process is in an env + # and has been launched by abspath without + # activating the env, python on PATH may not be sys.executable, + # but it should be. + cmd[0] = sys.executable + + ns = dict(connection_file=connection_file, + prefix=sys.prefix, + ) + + if kernel_resource_dir: + ns["resource_dir"] = kernel_resource_dir + + pat = re.compile(r'\{([A-Za-z0-9_]+)\}') + def from_ns(match): + """Get the key out of ns if it's there, otherwise no change.""" + return ns.get(match.group(1), match.group()) + + return [ pat.sub(from_ns, arg) for arg in cmd ] + +def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): + """Build a dictionary of arguments to pass to Popen""" + kwargs = {} + # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr + # are invalid. Unfortunately, there is in general no way to detect whether + # they are valid. The following two blocks redirect them to (temporary) + # pipes in certain important cases. + + # If this process has been backgrounded, our stdin is invalid. Since there + # is no compelling reason for the kernel to inherit our stdin anyway, we'll + # place this one safe and always redirect. + kwargs['stdin'] = PIPE + + # If this process in running on pythonw, we know that stdin, stdout, and + # stderr are all invalid. + redirect_out = sys.executable.endswith('pythonw.exe') + if redirect_out: + kwargs['stdout'] = kwargs['stderr'] = open(os.devnull, 'w') + + cmd = format_kernel_cmd(cmd_template, connection_file) + + kwargs['env'] = env = os.environ.copy() + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + + if extra_env: + env.update(extra_env) + + # TODO: where is this used? + independent = False + + if sys.platform == 'win32': + # Popen on Python 2 on Windows cannot handle unicode args or cwd + encoding = getdefaultencoding(prefer_stream=False) + kwargs['args'] = [cast_bytes_py2(c, encoding) for c in cmd] + if cwd: + kwargs['cwd'] = cast_bytes_py2(cwd, + sys.getfilesystemencoding() or 'ascii') + + from .win_interrupt import create_interrupt_event + # Create a Win32 event for interrupting the kernel + # and store it in an environment variable. + interrupt_event = create_interrupt_event() + env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) + # deprecated old env name: + env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] + + try: + from _winapi import DuplicateHandle, GetCurrentProcess, \ + DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP + except: + from _subprocess import DuplicateHandle, GetCurrentProcess, \ + DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP + # Launch the kernel process + if independent: + kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP + else: + pid = GetCurrentProcess() + handle = DuplicateHandle(pid, pid, pid, 0, + True, # Inheritable by new processes. + DUPLICATE_SAME_ACCESS) + env['JPY_PARENT_PID'] = str(int(handle)) + + else: + kwargs['args'] = cmd + kwargs['cwd'] = cwd + # Create a new session. + # This makes it easier to interrupt the kernel, + # because we want to interrupt the whole process group. + # We don't use setpgrp, which is known to cause problems for kernels starting + # certain interactive subprocesses, such as bash -i. + if six.PY3: + kwargs['start_new_session'] = True + else: + kwargs['preexec_fn'] = lambda: os.setsid() + if not independent: + env['JPY_PARENT_PID'] = str(os.getpid()) + + return kwargs + +class KernelLauncherBase(six.with_metaclass(ABCMeta, object)): + @abstractmethod + def launch(self): + """Launch the kernel.""" + + @abstractmethod + def is_alive(self): + """Check whether the kernel is currently alive (e.g. the process exists) + """ + pass + + @abstractmethod + def wait(self): + """Wait for the kernel process to exit. + """ + raise NotImplementedError() + + @abstractmethod + def signal(self, signum): + """Send a signal to the kernel.""" + pass + + def cleanup(self): + """Clean up any resources.""" + pass + +class PopenKernelLauncher(KernelLauncherBase): + popen = None + connection_file = None + connection_info = None + + def __init__(self, cmd_template, extra_env=None, cwd=None): + self.cmd_template = cmd_template + self.extra_env = extra_env + self.cwd = cwd + + def launch(self): + self.connection_file, self.connection_info = make_connection_file() + kwargs = build_popen_kwargs(self.cmd_template, self.connection_file, + self.extra_env, self.cwd) + self.popen = Popen(**kwargs) + return self.connection_info + + def poll(self): + return self.popen.poll() is None + + def wait(self): + return self.popen.wait() + + def send_signal(self, signum): + self.popen.send_signal(signum) + + def cleanup(self): + if self.connection_file: + os.unlink(self.connection_file) From b6050cc6658e5d6186c5b984575ee51a5cb615f6 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 30 Nov 2017 13:57:29 +0000 Subject: [PATCH 29/49] Integrate launcher classes into the discovery API --- jupyter_client/async_launcher.py | 44 +++++++++++++++++--------- jupyter_client/discovery.py | 54 ++++++++++++++++++++++++++------ jupyter_client/launcher2.py | 26 +++++++-------- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/jupyter_client/async_launcher.py b/jupyter_client/async_launcher.py index d62069249..0dc4df9b6 100644 --- a/jupyter_client/async_launcher.py +++ b/jupyter_client/async_launcher.py @@ -1,6 +1,7 @@ """Launch and control kernels using asyncio. """ from abc import ABC, abstractmethod +# noinspection PyCompatibility import asyncio from .launcher2 import make_connection_file, build_popen_kwargs @@ -9,12 +10,9 @@ class AsyncKernelLauncher(ABC): """Interface for async kernel launchers. This is very similar to the KernelLauncher interface, but its methods - are coroutines. + are asyncio coroutines. There is no poll method, but you can get a future + from the wait method and then poll it by checking ``future.done()``. """ - @abstractmethod - def launch(self): - """Launch the kernel.""" - @abstractmethod def wait(self): """Wait for the kernel process to exit. @@ -30,7 +28,13 @@ def cleanup(self): """Clean up any resources.""" pass + @abstractmethod + def get_connection_info(self): + """Return a dictionary of connection information""" + pass + +# noinspection PyCompatibility class AsyncPopenKernelLauncher(AsyncKernelLauncher): """Launch a kernel asynchronously in a subprocess. @@ -40,18 +44,21 @@ class AsyncPopenKernelLauncher(AsyncKernelLauncher): connection_file = None connection_info = None - def __init__(self, cmd_template, extra_env=None, cwd=None): - self.cmd_template = cmd_template - self.extra_env = extra_env - self.cwd = cwd + def __init__(self, process, connection_file, connection_info): + self.process = process + self.connection_file = connection_file + self.connection_info = connection_info + # __init__ can't be async, so this is the preferred constructor: + @classmethod @asyncio.coroutine - def launch(self): - self.connection_file, self.connection_info = make_connection_file() - kwargs = build_popen_kwargs(self.cmd_template, self.connection_file, - self.extra_env, self.cwd) + def launch(cls, cmd_template, extra_env, cwd): + connection_file, connection_info = make_connection_file() + kwargs = build_popen_kwargs(cmd_template, connection_file, + extra_env, cwd) args = kwargs.pop('args') - self.process = yield from asyncio.create_subprocess_exec(*args, **kwargs) + p = yield from asyncio.create_subprocess_exec(*args, **kwargs) + return cls(p, connection_file, connection_info) @asyncio.coroutine def wait(self): @@ -65,6 +72,12 @@ def send_signal(self, signum): def cleanup(self): super().cleanup() + @asyncio.coroutine + def get_connection_info(self): + return self.connection_info + + +# noinspection PyCompatibility class AsyncLauncherWrapper(AsyncKernelLauncher): """Wrap a blocking KernelLauncher to be used asynchronously. @@ -93,3 +106,6 @@ def send_signal(self, signum): def cleanup(self): return (yield from self.in_default_executor(self.wrapped.cleanup)) + @asyncio.coroutine + def get_connection_info(self): + return (yield from self.in_default_executor(self.wrapped.get_connection_info)) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 2bfe92b2a..592dcc77d 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -4,7 +4,7 @@ import six from .kernelspec import KernelSpecManager -from .manager import KernelManager +from .launcher2 import PopenKernelLauncher log = logging.getLogger(__name__) @@ -17,13 +17,25 @@ def find_kernels(self): pass @abstractmethod - def make_manager(self, name): - """Make and return a KernelManager instance to start a specified kernel + def launch(self, name, cwd=None): + """Launch a kernel, return an object with the KernelLauncher interface. name will be one of the kernel names produced by find_kernels() + + This method launches and manages the kernel in a blocking manner. """ pass + def launch_async(self, name, cwd=None): + """Launch a kernel asynchronously using asyncio. + + name will be one of the kernel names produced by find_kernels() + + This method should return an asyncio future, which resolves to an object + with the AsyncKernelLauncher interface. + """ + raise NotImplementedError() + class KernelSpecProvider(KernelProviderBase): """Offers kernel types from installed kernelspec directories. """ @@ -42,10 +54,16 @@ def find_kernels(self): 'argv': spec.argv, } - def make_manager(self, name): + def launch(self, name, cwd=None): spec = self.ksm.get_kernel_spec(name) - return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) + return PopenKernelLauncher(cmd_template=spec.argv, + extra_env=spec.env, cwd=cwd) + def launch_async(self, name, cwd=None): + from .async_launcher import AsyncPopenKernelLauncher + spec = self.ksm.get_kernel_spec(name) + return AsyncPopenKernelLauncher.launch(cmd_template=spec.argv, + extra_env=spec.env, cwd=cwd) class IPykernelProvider(KernelProviderBase): """Offers a kernel type using the Python interpreter it's running in. @@ -77,12 +95,20 @@ def find_kernels(self): 'argv': info['spec']['argv'], } - def make_manager(self, name): + def launch(self, name, cwd=None): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") - return KernelManager(kernel_cmd=info['spec']['argv']) + return PopenKernelLauncher(cmd_template=info['spec']['argv'], + extra_env={}, cwd=cwd) + def launch_async(self, name, cwd=None): + from .async_launcher import AsyncPopenKernelLauncher + info = self._check_for_kernel() + if info is None: + raise Exception("ipykernel is not importable") + return AsyncPopenKernelLauncher.launch( + cmd_template=info['spec']['argv'], extra_env={}, cwd=cwd) class KernelFinder(object): """Manages a collection of kernel providers to find available kernel types @@ -122,10 +148,18 @@ def find_kernels(self): id = provider.id + '/' + kid yield id, attributes - def make_manager(self, name): - """Make a KernelManager instance for a given kernel type. + def launch(self, name, cwd=None): + """Launch a kernel of a given kernel type. + """ + provider_id, kernel_id = name.split('/', 1) + for provider in self.providers: + if provider_id == provider.id: + return provider.launch(kernel_id, cwd) + + def launch_async(self, name, cwd=None): + """Launch a kernel of a given kernel type, using asyncio. """ provider_id, kernel_id = name.split('/', 1) for provider in self.providers: if provider_id == provider.id: - return provider.make_manager(kernel_id) + return provider.launch_async(kernel_id, cwd) diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py index 0b2d8b6e3..80f3bb664 100644 --- a/jupyter_client/launcher2.py +++ b/jupyter_client/launcher2.py @@ -96,9 +96,6 @@ def make_connection_file(ip=None, transport='tcp'): Parameters ---------- - fname : unicode - The path to the file to write - ip : str, optional The ip address the kernel will bind to. @@ -198,9 +195,11 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] try: + # noinspection PyUnresolvedReferences from _winapi import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP except: + # noinspection PyUnresolvedReferences from _subprocess import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP # Launch the kernel process @@ -230,11 +229,7 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): return kwargs -class KernelLauncherBase(six.with_metaclass(ABCMeta, object)): - @abstractmethod - def launch(self): - """Launch the kernel.""" - +class KernelLauncher(six.with_metaclass(ABCMeta, object)): @abstractmethod def is_alive(self): """Check whether the kernel is currently alive (e.g. the process exists) @@ -256,22 +251,20 @@ def cleanup(self): """Clean up any resources.""" pass -class PopenKernelLauncher(KernelLauncherBase): - popen = None - connection_file = None - connection_info = None + @abstractmethod + def get_connection_info(self): + """Return a dictionary of connection information""" + pass +class PopenKernelLauncher(KernelLauncher): def __init__(self, cmd_template, extra_env=None, cwd=None): self.cmd_template = cmd_template self.extra_env = extra_env self.cwd = cwd - - def launch(self): self.connection_file, self.connection_info = make_connection_file() kwargs = build_popen_kwargs(self.cmd_template, self.connection_file, self.extra_env, self.cwd) self.popen = Popen(**kwargs) - return self.connection_info def poll(self): return self.popen.poll() is None @@ -285,3 +278,6 @@ def send_signal(self, signum): def cleanup(self): if self.connection_file: os.unlink(self.connection_file) + + def get_connection_info(self): + return self.connection_info From fd2631e74ad4fc2be51a48596f7fff08a817cbb1 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 30 Nov 2017 15:09:20 +0000 Subject: [PATCH 30/49] Fix up discovery tests --- jupyter_client/tests/test_discovery.py | 29 +++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index 9d7833ba3..fd5269592 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -1,7 +1,7 @@ import sys -from jupyter_client import KernelManager from jupyter_client import discovery +from jupyter_client.launcher2 import KernelLauncher def test_ipykernel_provider(): import ipykernel # Fail clearly if ipykernel not installed @@ -20,13 +20,32 @@ class DummyKernelProvider(discovery.KernelProviderBase): def find_kernels(self): yield 'sample', {'argv': ['dummy_kernel']} - def make_manager(self, name): - return KernelManager(kernel_cmd=['dummy_kernel']) + def launch(self, name, cwd=None): + return DummyKernelLauncher() + +class DummyKernelLauncher(KernelLauncher): + def is_alive(self): + """Check whether the kernel is currently alive (e.g. the process exists) + """ + return True + + def wait(self): + """Wait for the kernel process to exit. + """ + return 0 + + def signal(self, signum): + """Send a signal to the kernel.""" + pass + + def get_connection_info(self): + """Return a dictionary of connection information""" + return {} def test_meta_kernel_finder(): kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) assert list(kf.find_kernels()) == \ [('dummy/sample', {'argv': ['dummy_kernel']})] - manager = kf.make_manager('dummy/sample') - assert manager.kernel_cmd == ['dummy_kernel'] + launcher = kf.launch('dummy/sample') + assert isinstance(launcher, DummyKernelLauncher) From 2955d355907fe70bf31bcb1a46d01f955c44812f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 4 Dec 2017 18:57:19 +0000 Subject: [PATCH 31/49] Start making new Manager and Client interfaces --- jupyter_client/channels.py | 4 +- jupyter_client/client2.py | 891 ++++++++++++++++++++++++++ jupyter_client/launcher2.py | 8 +- jupyter_client/manager2.py | 246 +++++++ jupyter_client/tests/test_manager2.py | 92 +++ 5 files changed, 1237 insertions(+), 4 deletions(-) create mode 100644 jupyter_client/client2.py create mode 100644 jupyter_client/manager2.py create mode 100644 jupyter_client/tests/test_manager2.py diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index dd9906723..e24ba8311 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -193,9 +193,9 @@ def close(self): self.socket = None def call_handlers(self, since_last_heartbeat): - """This method is called in the ioloop thread when a message arrives. + """This method is called in the ioloop thread when the heartbeat fails. - Subclasses should override this method to handle incoming messages. + Subclasses should override this method to handle heartbeat failure. It is important to remember that this method is called in the thread so that some logic must be done to ensure that the application level handlers are called in the application thread. diff --git a/jupyter_client/client2.py b/jupyter_client/client2.py new file mode 100644 index 000000000..02041699e --- /dev/null +++ b/jupyter_client/client2.py @@ -0,0 +1,891 @@ +"""Base class to manage the interaction with a running kernel""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from __future__ import absolute_import + +import atexit +import errno +from functools import partial +from getpass import getpass +from six.moves import input +import sys +from threading import Thread +import time +import zmq +from zmq import ZMQError +from zmq.eventloop import ioloop, zmqstream + +from ipython_genutils.py3compat import string_types, iteritems +from traitlets.log import get_logger as get_app_logger +from .channels import major_protocol_version, HBChannel +from .session import Session + +try: + monotonic = time.monotonic +except AttributeError: + # py2 + monotonic = time.time # close enough + +try: + TimeoutError +except NameError: + # py2 + TimeoutError = RuntimeError + + +channel_socket_types = { + 'hb' : zmq.REQ, + 'shell' : zmq.DEALER, + 'iopub' : zmq.SUB, + 'stdin' : zmq.DEALER, + 'control': zmq.DEALER, +} + +# some utilities to validate message structure, these might get moved elsewhere +# if they prove to have more generic utility + +def validate_string_dict(dct): + """Validate that the input is a dict with string keys and values. + + Raises ValueError if not.""" + for k, v in iteritems(dct): + if not isinstance(k, string_types): + raise ValueError('key %r in dict must be a string' % k) + if not isinstance(v, string_types): + raise ValueError('value %r in dict must be a string' % v) + +def inherit_docstring(cls): + def decorator(func): + doc = getattr(cls, func.__name__).__doc__ + func.__doc__ = doc + return func + return decorator + +class KernelClient2(): + """Communicates with a single kernel on any host via zmq channels. + + The messages that can be sent are exposed as methods of the + client (KernelClient2.execute, complete, history, etc.). These methods only + send the message, they don't wait for a reply. To get results, use e.g. + :meth:`get_shell_msg` to fetch messages from the shell channel. + """ + hb_channel = None + + def __init__(self, connection_info, manager=None, use_heartbeat=True): + self.connection_info = connection_info + self.manager = manager + self.using_heartbeat = use_heartbeat and (manager is not None) + self.context = zmq.Context.instance() + self.session = Session(key=connection_info['key'].encode('ascii'), + signature_scheme=connection_info['signature_scheme']) + self.log = get_app_logger() + + identity = self.session.bsession + self.iopub_socket = self._create_connected_socket('iopub', identity) + self.iopub_socket.setsockopt(zmq.SUBSCRIBE, b'') + self.shell_socket = self._create_connected_socket('shell', identity) + self.stdin_socket = self._create_connected_socket('stdin', identity) + self.control_socket = self._create_connected_socket('control', identity) + if self.using_heartbeat: + self.hb_monitor = HBChannel(context=self.context, + address=self._make_url('hb')) + self.hb_monitor.start() + + @property + def owned_kernel(self): + """True if this client 'owns' the kernel, i.e. started it.""" + return self.manager is not None + + # flag for whether execute requests should be allowed to call raw_input: + allow_stdin = True + + def _make_url(self, channel): + """Make a ZeroMQ URL for a given channel.""" + transport = self.connection_info['transport'] + ip = self.connection_info['ip'] + port = self.connection_info['%s_port' % channel] + + if transport == 'tcp': + return "tcp://%s:%i" % (ip, port) + else: + return "%s://%s-%s" % (transport, ip, port) + + def _create_connected_socket(self, channel, identity=None): + """Create a zmq Socket and connect it to the kernel.""" + url = self._make_url(channel) + socket_type = channel_socket_types[channel] + self.log.debug("Connecting to: %s" % url) + sock = self.context.socket(socket_type) + # set linger to 1s to prevent hangs at exit + sock.linger = 1000 + if identity: + sock.identity = identity + sock.connect(url) + return sock + + def is_alive(self): + if self.owned_kernel: + return self.manager.is_alive() + elif self.using_heartbeat: + return self.hb_monitor.is_beating() + else: + return True # Fingers crossed + + def _send(self, socket, msg): + self.session.send(socket, msg) + + # Methods to send specific messages on channels + def execute(self, code, silent=False, store_history=True, + user_expressions=None, allow_stdin=None, stop_on_error=True, + _header=None): + """Execute code in the kernel. + + Parameters + ---------- + code : str + A string of code in the kernel's language. + + silent : bool, optional (default False) + If set, the kernel will execute the code as quietly possible, and + will force store_history to be False. + + store_history : bool, optional (default True) + If set, the kernel will store command history. This is forced + to be False if silent is True. + + user_expressions : dict, optional + A dict mapping names to expressions to be evaluated in the user's + dict. The expression values are returned as strings formatted using + :func:`repr`. + + allow_stdin : bool, optional (default self.allow_stdin) + Flag for whether the kernel can send stdin requests to frontends. + + Some frontends (e.g. the Notebook) do not support stdin requests. + If raw_input is called from code executed from such a frontend, a + StdinNotImplementedError will be raised. + + stop_on_error: bool, optional (default True) + Flag whether to abort the execution queue, if an exception is encountered. + + Returns + ------- + The msg_id of the message sent. + """ + if user_expressions is None: + user_expressions = {} + if allow_stdin is None: + allow_stdin = self.allow_stdin + + # Don't waste network traffic if inputs are invalid + if not isinstance(code, string_types): + raise ValueError('code %r must be a string' % code) + validate_string_dict(user_expressions) + + # Create class for content/msg creation. Related to, but possibly + # not in Session. + content = dict(code=code, silent=silent, store_history=store_history, + user_expressions=user_expressions, + allow_stdin=allow_stdin, stop_on_error=stop_on_error + ) + msg = self.session.msg('execute_request', content, header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def complete(self, code, cursor_pos=None, _header=None): + """Tab complete text in the kernel's namespace. + + Parameters + ---------- + code : str + The context in which completion is requested. + Can be anything between a variable name and an entire cell. + cursor_pos : int, optional + The position of the cursor in the block of code where the completion was requested. + Default: ``len(code)`` + + Returns + ------- + The msg_id of the message sent. + """ + if cursor_pos is None: + cursor_pos = len(code) + content = dict(code=code, cursor_pos=cursor_pos) + msg = self.session.msg('complete_request', content, header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def inspect(self, code, cursor_pos=None, detail_level=0, _header=None): + """Get metadata information about an object in the kernel's namespace. + + It is up to the kernel to determine the appropriate object to inspect. + + Parameters + ---------- + code : str + The context in which info is requested. + Can be anything between a variable name and an entire cell. + cursor_pos : int, optional + The position of the cursor in the block of code where the info was requested. + Default: ``len(code)`` + detail_level : int, optional + The level of detail for the introspection (0-2) + + Returns + ------- + The msg_id of the message sent. + """ + if cursor_pos is None: + cursor_pos = len(code) + content = dict(code=code, cursor_pos=cursor_pos, + detail_level=detail_level, + ) + msg = self.session.msg('inspect_request', content, header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def history(self, raw=True, output=False, hist_access_type='range', + _header=None, **kwargs): + """Get entries from the kernel's history list. + + Parameters + ---------- + raw : bool + If True, return the raw input. + output : bool + If True, then return the output as well. + hist_access_type : str + 'range' (fill in session, start and stop params), 'tail' (fill in n) + or 'search' (fill in pattern param). + + session : int + For a range request, the session from which to get lines. Session + numbers are positive integers; negative ones count back from the + current session. + start : int + The first line number of a history range. + stop : int + The final (excluded) line number of a history range. + + n : int + The number of lines of history to get for a tail request. + + pattern : str + The glob-syntax pattern for a search request. + + Returns + ------- + The ID of the message sent. + """ + if hist_access_type == 'range': + kwargs.setdefault('session', 0) + kwargs.setdefault('start', 0) + content = dict(raw=raw, output=output, + hist_access_type=hist_access_type, + **kwargs) + msg = self.session.msg('history_request', content, header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def kernel_info(self, _header=None): + """Request kernel info + + Returns + ------- + The msg_id of the message sent + """ + msg = self.session.msg('kernel_info_request', header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def comm_info(self, target_name=None, _header=None): + """Request comm info + + Returns + ------- + The msg_id of the message sent + """ + if target_name is None: + content = {} + else: + content = dict(target_name=target_name) + msg = self.session.msg('comm_info_request', content, header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def _handle_kernel_info_reply(self, msg): + """handle kernel info reply + + sets protocol adaptation version. This might + be run from a separate thread. + """ + adapt_version = int(msg['content']['protocol_version'].split('.')[0]) + if adapt_version != major_protocol_version: + self.session.adapt_version = adapt_version + + def shutdown(self, restart=False, _header=None): + """Request an immediate kernel shutdown. + + Upon receipt of the (empty) reply, client code can safely assume that + the kernel has shut down and it's safe to forcefully terminate it if + it's still alive. + + The kernel will send the reply via a function registered with Python's + atexit module, ensuring it's truly done as the kernel is done with all + normal operation. + + Returns + ------- + The msg_id of the message sent + """ + # Send quit message to kernel. Once we implement kernel-side setattr, + # this should probably be done that way, but for now this will do. + msg = self.session.msg('shutdown_request', {'restart': restart}, + header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def is_complete(self, code, _header=None): + """Ask the kernel whether some code is complete and ready to execute.""" + msg = self.session.msg('is_complete_request', {'code': code}, + header=_header) + self._send(self.shell_socket, msg) + return msg['header']['msg_id'] + + def interrupt(self, _header=None): + """Send an interrupt message/signal to the kernel""" + mode = self.connection_info.get('interrupt_mode', 'signal') + if mode == 'message': + msg = self.session.msg("interrupt_request", content={}, + header=_header) + self._send(self.control_socket, msg) + return msg['header']['msg_id'] + elif self.owned_kernel: + self.manager.interrupt_kernel() + else: + self.log.warning("Can't send signal to non-owned kernel") + + def input(self, string, parent=None, _header=None): + """Send a string of raw input to the kernel. + + This should only be called in response to the kernel sending an + ``input_request`` message on the stdin channel. + """ + content = dict(value=string) + msg = self.session.msg('input_reply', content, + header=_header, parent=parent) + self._send(self.stdin_socket, msg) + + +def reqrep(meth): + def wrapped(self, *args, **kwargs): + reply = kwargs.pop('reply', False) + timeout = kwargs.pop('timeout', None) + msg_id = meth(self, *args, **kwargs) + if not reply: + return msg_id + + return self._recv_reply(msg_id, timeout=timeout) + + if not meth.__doc__: + # python -OO removes docstrings, + # so don't bother building the wrapped docstring + return wrapped + + basedoc, _ = meth.__doc__.split('Returns\n', 1) + parts = [basedoc.strip()] + if 'Parameters' not in basedoc: + parts.append(""" + Parameters + ---------- + """) + parts.append(""" + reply: bool (default: False) + Whether to wait for and return reply + timeout: float or None (default: None) + Timeout to use when waiting for a reply + + Returns + ------- + msg_id: str + The msg_id of the request sent, if reply=False (default) + reply: dict + The reply message for this request, if reply=True + """) + wrapped.__doc__ = '\n'.join(parts) + return wrapped + + +class BlockingKernelClient2(KernelClient2): + """A KernelClient with blocking APIs + + ``get_[channel]_msg()`` methods wait for and return messages on channels, + returning None if no message arrives within ``timeout`` seconds. + """ + + def _recv(self, socket): + """Receive and parse a message""" + msg = socket.recv_multipart() + ident,smsg = self.session.feed_identities(msg) + return self.session.deserialize(smsg) + + def _get_msg(self, socket, block=True, timeout=None): + if block: + if timeout is not None: + timeout *= 1000 # seconds to ms + ready = self.shell_socket.poll(timeout) + else: + ready = self.shell_socket.poll(timeout=0) + + if ready: + return self._recv(socket) + + def get_shell_msg(self, block=True, timeout=None): + """Get a message from the shell channel""" + return self._get_msg(self.shell_socket, block, timeout) + + def get_iopub_msg(self, block=True, timeout=None): + """Get a message from the iopub channel""" + return self._get_msg(self.iopub_socket, block, timeout) + + def get_stdin_msg(self, block=True, timeout=None): + """Get a message from the stdin channel""" + return self._get_msg(self.stdin_socket, block, timeout) + + def wait_for_ready(self, timeout=None): + """Waits for a response when a client is blocked + + - Sets future time for timeout + - Blocks on shell channel until a message is received + - Exit if the kernel has died + - If client times out before receiving a message from the kernel, send RuntimeError + - Flush the IOPub channel + """ + if timeout is None: + abs_timeout = float('inf') + else: + abs_timeout = time.time() + timeout + + if not self.owned_kernel: + # This Client was not created by a KernelManager, + # so wait for kernel to become responsive to heartbeats + # before checking for kernel_info reply + while not self.is_alive(): + if time.time() > abs_timeout: + raise RuntimeError( + "Kernel didn't respond to heartbeats in %d seconds and timed out" % timeout) + time.sleep(0.2) + + self.kernel_info(reply=False) + + # Wait for kernel info reply on shell channel + while True: + msg = self.get_shell_msg(timeout=1) + if msg and msg['msg_type'] == 'kernel_info_reply': + self._handle_kernel_info_reply(msg) + break + + if not self.is_alive(): + raise RuntimeError('Kernel died before replying to kernel_info') + + # Check if current time is ready check time plus timeout + if time.time() > abs_timeout: + raise RuntimeError( + "Kernel didn't respond in %d seconds" % timeout) + + # Flush IOPub channel + while True: + msg = self.get_iopub_msg(block=True, timeout=0.2) + if msg is None: + break + + def _recv_reply(self, msg_id, timeout=None): + """Receive and return the reply for a given request""" + deadline = None + if timeout is not None: + deadline = monotonic() + timeout + while True: + if timeout is not None: + timeout = max(0, deadline - monotonic()) + reply = self.get_shell_msg(timeout=timeout) + if reply is None: + raise TimeoutError("Timeout waiting for reply") + elif reply['parent_header'].get('msg_id') != msg_id: + # not my reply, someone may have forgotten to retrieve theirs + continue + return reply + + execute = reqrep(KernelClient2.execute) + history = reqrep(KernelClient2.history) + complete = reqrep(KernelClient2.complete) + inspect = reqrep(KernelClient2.inspect) + kernel_info = reqrep(KernelClient2.kernel_info) + comm_info = reqrep(KernelClient2.comm_info) + shutdown = reqrep(KernelClient2.shutdown) + + @inherit_docstring(KernelClient2) + def interrupt(self, reply=False, timeout=None): + msg_id = super(BlockingKernelClient2, self).interrupt() + if reply and msg_id: + return self._recv_reply(msg_id, timeout=timeout) + else: + return msg_id + + def _stdin_hook_default(self, msg): + """Handle an input request""" + content = msg['content'] + if content.get('password', False): + prompt = getpass + else: + prompt = input + + try: + raw_data = prompt(content["prompt"]) + except EOFError: + # turn EOFError into EOF character + raw_data = '\x04' + except KeyboardInterrupt: + sys.stdout.write('\n') + return + + # only send stdin reply if there *was not* another request + # or execution finished while we were reading. + if not (self.stdin_socket.poll(timeout=0) + or self.shell_socket.poll(timeout=0)): + self.input(raw_data) + + def _output_hook_default(self, msg): + """Default hook for redisplaying plain-text output""" + msg_type = msg['header']['msg_type'] + content = msg['content'] + if msg_type == 'stream': + stream = getattr(sys, content['name']) + stream.write(content['text']) + elif msg_type in ('display_data', 'execute_result'): + sys.stdout.write(content['data'].get('text/plain', '')) + elif msg_type == 'error': + print('\n'.join(content['traceback']), file=sys.stderr) + + def _output_hook_kernel(self, session, socket, parent_header, msg): + """Output hook when running inside an IPython kernel + + adds rich output support. + """ + msg_type = msg['header']['msg_type'] + if msg_type in ('display_data', 'execute_result', 'error'): + session.send(socket, msg_type, msg['content'], parent=parent_header) + else: + self._output_hook_default(msg) + + def execute_interactive(self, code, silent=False, store_history=True, + user_expressions=None, allow_stdin=None, + stop_on_error=True, + timeout=None, output_hook=None, stdin_hook=None, + ): + """Execute code in the kernel interactively + + Output will be redisplayed, and stdin prompts will be relayed as well. + If an IPython kernel is detected, rich output will be displayed. + + You can pass a custom output_hook callable that will be called + with every IOPub message that is produced instead of the default redisplay. + + .. versionadded:: 5.0 + + Parameters + ---------- + code : str + A string of code in the kernel's language. + + silent : bool, optional (default False) + If set, the kernel will execute the code as quietly possible, and + will force store_history to be False. + + store_history : bool, optional (default True) + If set, the kernel will store command history. This is forced + to be False if silent is True. + + user_expressions : dict, optional + A dict mapping names to expressions to be evaluated in the user's + dict. The expression values are returned as strings formatted using + :func:`repr`. + + allow_stdin : bool, optional (default self.allow_stdin) + Flag for whether the kernel can send stdin requests to frontends. + + Some frontends (e.g. the Notebook) do not support stdin requests. + If raw_input is called from code executed from such a frontend, a + StdinNotImplementedError will be raised. + + stop_on_error: bool, optional (default True) + Flag whether to abort the execution queue, if an exception is encountered. + + timeout: float or None (default: None) + Timeout to use when waiting for a reply + + output_hook: callable(msg) + Function to be called with output messages. + If not specified, output will be redisplayed. + + stdin_hook: callable(msg) + Function to be called with stdin_request messages. + If not specified, input/getpass will be called. + + Returns + ------- + reply: dict + The reply message for this request + """ + if allow_stdin is None: + allow_stdin = self.allow_stdin + msg_id = self.execute(code, + silent=silent, + store_history=store_history, + user_expressions=user_expressions, + allow_stdin=allow_stdin, + stop_on_error=stop_on_error, + ) + if stdin_hook is None: + stdin_hook = self._stdin_hook_default + if output_hook is None: + # detect IPython kernel + if 'IPython' in sys.modules: + from IPython import get_ipython + ip = get_ipython() + in_kernel = getattr(ip, 'kernel', False) + if in_kernel: + output_hook = partial( + self._output_hook_kernel, + ip.display_pub.session, + ip.display_pub.pub_socket, + ip.display_pub.parent_header, + ) + if output_hook is None: + # default: redisplay plain-text outputs + output_hook = self._output_hook_default + + # set deadline based on timeout + timeout_ms = None + if timeout is not None: + deadline = monotonic() + timeout + else: + deadline = None + + poller = zmq.Poller() + poller.register(self.iopub_socket, zmq.POLLIN) + if allow_stdin: + poller.register(self.stdin_socket, zmq.POLLIN) + + # wait for output and redisplay it + while True: + if deadline is not None: + timeout = max(0, deadline - monotonic()) + timeout_ms = 1e3 * timeout + events = dict(poller.poll(timeout_ms)) + if not events: + raise TimeoutError("Timeout waiting for output") + if self.stdin_socket in events: + req = self.get_stdin_msg(timeout=0) + stdin_hook(req) + continue + if self.iopub_socket not in events: + continue + + msg = self.get_iopub_msg(timeout=0) + + if msg['parent_header'].get('msg_id') != msg_id: + # not from my request + continue + output_hook(msg) + + # stop on idle + if msg['header']['msg_type'] == 'status' and \ + msg['content']['execution_state'] == 'idle': + break + + # output is done, get the reply + if timeout is not None: + timeout = max(0, deadline - monotonic()) + return self._recv_reply(msg_id, timeout=timeout) + + +class IOLoopKernelClient2(KernelClient2): + """Runs a zmq IOLoop to send and receive messages. + + Use ClientInThread to run this in a separate thread alongside your + application. + """ + def __init__(self, **kwargs): + super(IOLoopKernelClient2, self).__init__(**kwargs) + self.ioloop = ioloop.IOLoop.instance() + self.handlers = { + 'iopub': [], + 'shell': [], + 'stdin': [], + } + self.shell_stream = zmqstream.ZMQStream(self.shell_socket, self.ioloop) + self.shell_stream.on_recv(partial(self._handle_recv, 'shell')) + self.iopub_stream = zmqstream.ZMQStream(self.iopub_socket, self.ioloop) + self.iopub_stream.on_recv(partial(self._handle_recv, 'iopub')) + self.stdin_stream = zmqstream.ZMQStream(self.stdin_socket, self.ioloop) + self.stdin_stream.on_recv(partial(self._handle_recv, 'stdin')) + + _inspect = None + + def _handle_recv(self, channel, msg): + """Callback for stream.on_recv. + + Unpacks message, and calls handlers with it. + """ + ident,smsg = self.session.feed_identities(msg) + msg = self.session.deserialize(smsg) + # let client inspect messages + if self._inspect: + self._inspect(msg) + self._call_handlers(channel, msg) + + def _call_handlers(self, channel, msg): + for handler in self.handlers[channel]: + try: + handler() + except Exception as e: + self.log.error("Exception from message handler %r", handler, + exc_info=e) + + def add_handler(self, channel, handler): + self.handlers[channel].append(handler) + + def remove_handler(self, channel, handler): + self.handlers[channel].remove(handler)# + +class ClientInThread(Thread): + client = None + _exiting = False + + def __init__(self, connection_info, manager=None): + super(ClientInThread, self).__init__() + self.daemon = True + self.connection_info = connection_info + self.manager = manager + + @staticmethod + @atexit.register + def _notice_exit(): + ClientInThread._exiting = True + + def run(self): + """Run my loop, ignoring EINTR events in the poller""" + self.client = IOLoopKernelClient2() + while True: + try: + self.client.ioloop.start() + except ZMQError as e: + if e.errno == errno.EINTR: + continue + else: + raise + except Exception: + if self._exiting: + break + else: + raise + else: + break + + @property + def ioloop(self): + if self.client: + return self.client.ioloop + + def stop(self): + """Stop the channel's event loop and join its thread. + + This calls :meth:`~threading.Thread.join` and returns when the thread + terminates. :class:`RuntimeError` will be raised if + :meth:`~threading.Thread.start` is called again. + """ + if self.client is not None: + self.client.ioloop.stop() + self.join() + self.close() + + def close(self): + if self.client is not None: + try: + self.client.ioloop.close(all_fds=True) + except Exception: + pass + + def _call_in_thread(self, function, *args, **kwargs): + return self.ioloop.add_callback(function, *args, **kwargs) + + # Client messaging methods -------------------------------- + # These send as much work as possible to the IO thread, but we generate + # the header in the calling thread so we can return the message ID. + + @inherit_docstring(KernelClient2) + def execute(self, *args, **kwargs): + hdr = self.client.session.msg_header('execute_request') + self._call_in_thread(self.client.execute, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def complete(self, *args, **kwargs): + hdr = self.client.session.msg_header('complete_request') + self._call_in_thread(self.client.complete, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def inspect(self, *args, **kwargs): + hdr = self.client.session.msg_header('inspect_request') + self._call_in_thread(self.client.inspect, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def history(self, *args, **kwargs): + hdr = self.client.session.msg_header('history_request') + self._call_in_thread(self.client.history, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def kernel_info(self, _header=None): + hdr = self.client.session.msg_header('kernel_info_request') + self._call_in_thread(self.client.kernel_info, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def comm_info(self, target_name=None, _header=None): + hdr = self.client.session.msg_header('comm_info_request') + self._call_in_thread(self.client.comm_info, target_name, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def shutdown(self, restart=False, _header=None): + hdr = self.client.session.msg_header('shutdown_request') + self._call_in_thread(self.client.shutdown, restart, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def is_complete(self, code, _header=None): + hdr = self.client.session.msg_header('is_complete_request') + self._call_in_thread(self.client.is_complete, code, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def interrupt(self, _header=None): + mode = self.connection_info.get('interrupt_mode', 'signal') + if mode == 'message': + hdr = self.client.session.msg_header('is_complete_request') + self._call_in_thread(self.client.interrupt, _header=hdr) + return hdr['msg_id'] + else: + self.client.interrupt() + + @inherit_docstring(KernelClient2) + def input(self, string, parent=None): + hdr = self.client.session.msg_header('input_reply') + self._call_in_thread(self.client.is_complete, string, + parent=parent, _header=hdr) + return hdr['msg_id'] diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py index 80f3bb664..7edfbf80b 100644 --- a/jupyter_client/launcher2.py +++ b/jupyter_client/launcher2.py @@ -13,8 +13,9 @@ from ipython_genutils.encoding import getdefaultencoding from ipython_genutils.py3compat import cast_bytes_py2 -from .localinterfaces import localhost from jupyter_core.paths import jupyter_runtime_dir +from jupyter_core.utils import ensure_dir_exists +from .localinterfaces import localhost def new_key(): """Generate a new random key string. @@ -105,7 +106,9 @@ def make_connection_file(ip=None, transport='tcp'): if not ip: ip = localhost() - fname = os.path.join(jupyter_runtime_dir(), 'kernel-%s.json' % new_key()) + runtime_dir = jupyter_runtime_dir() + ensure_dir_exists(runtime_dir) + fname = os.path.join(runtime_dir, 'kernel-%s.json' % new_key()) cfg = random_ports(ip=ip, transport=transport) cfg['ip'] = ip @@ -173,6 +176,7 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): env.pop('PYTHONEXECUTABLE', None) if extra_env: + print(extra_env) env.update(extra_env) # TODO: where is this used? diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py new file mode 100644 index 000000000..96f0bdbb1 --- /dev/null +++ b/jupyter_client/manager2.py @@ -0,0 +1,246 @@ +"""Base class to manage a running kernel""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from __future__ import absolute_import + +from contextlib import contextmanager +import os +import signal +from subprocess import Popen +import sys +import time + +from traitlets.log import get_logger as get_app_logger + +from .launcher2 import make_connection_file, build_popen_kwargs +from .localinterfaces import is_local_ip, local_ips, localhost + +class KernelManager2(object): + """Manages a single kernel in a subprocess on this host. + + This version starts kernels with Popen to listen on TCP sockets. + + Parameters + ---------- + + kernel_cmd : list of str + The Popen command template to launch the kernel + cwd : str + The working directory to launch the kernel in + extra_env : dict, optional + Dictionary of environment variables to update the existing environment + ip : str, optional + Set the kernel\'s IP address [default localhost]. + If the IP address is something other than localhost, then + Consoles on other machines will be able to connect + to the Kernel, so be careful! + """ + transport = 'tcp' + + # The kernel process with which the KernelManager is communicating. + # generally a Popen instance + kernel = None + + # The dictionary of info to connect to the kernel, and the file storing it + connection_info = None + connection_file = None + + def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): + self.kernel_cmd = kernel_cmd + self.cwd = cwd + self.extra_env = extra_env + if ip is None: + ip = localhost() + self.ip = ip + self.log = get_app_logger() + + def start_kernel(self): + """Starts a kernel on this host in a separate process. + """ + if self.transport == 'tcp' and not is_local_ip(self.ip): + raise RuntimeError("Can only launch a kernel on a local interface. " + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % local_ips() + ) + + self.connection_file, self.connection_info = \ + make_connection_file(self.ip, self.transport) + + kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, + self.extra_env, self.cwd) + + # launch the kernel subprocess + self.log.debug("Starting kernel: %s", kw['args']) + self.kernel = Popen(**kw) + + def finish_shutdown(self, timeout=5.0, pollinterval=0.1): + """Wait for kernel shutdown, then kill process if it doesn't shutdown. + + This does not send shutdown requests - use :meth:`.KernelClient2.shutdown` + first. + """ + for i in range(int(timeout/pollinterval)): + if self.is_alive(): + time.sleep(pollinterval) + else: + break + else: + # OK, we've waited long enough. + if self.has_kernel: + self.log.debug("Kernel is taking too long to finish, killing") + self._kill_kernel() + + def cleanup(self): + """Clean up resources when the kernel is shut down""" + if self.connection_file: + # cleanup connection files on full shutdown of kernel we started + try: + os.remove(self.connection_file) + except (IOError, OSError, AttributeError): + pass + self.connection_file = None + + @property + def has_kernel(self): + """Has a kernel been started that we are managing.""" + return self.kernel is not None + + def _kill_kernel(self): + """Kill the running kernel. + + This is a private method, callers should use shutdown_kernel(now=True). + """ + if self.has_kernel: + + # Signal the kernel to terminate (sends SIGKILL on Unix and calls + # TerminateProcess() on Win32). + try: + self.kernel.kill() + except OSError as e: + # In Windows, we will get an Access Denied error if the process + # has already terminated. Ignore it. + if sys.platform == 'win32': + if e.winerror != 5: + raise + # On Unix, we may get an ESRCH error if the process has already + # terminated. Ignore it. + else: + from errno import ESRCH + if e.errno != ESRCH: + raise + + # Block until the kernel terminates. + self.kernel.wait() + self.kernel = None + else: + raise RuntimeError("Cannot kill kernel. No kernel is running!") + + def interrupt_kernel(self): + """Interrupts the kernel by sending it a signal. + + Unlike ``signal_kernel``, this operation is well supported on all + platforms. + + Kernels may ask for interrupts to be delivered by a message rather than + a signal. This method does *not* handle that. Use KernelClient.interrupt + to send a message or a signal as appropriate. + """ + if self.has_kernel: + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) + else: + self.signal_kernel(signal.SIGINT) + else: + raise RuntimeError("Cannot interrupt kernel. No kernel is running!") + + def signal_kernel(self, signum): + """Sends a signal to the process group of the kernel (this + usually includes the kernel and any subprocesses spawned by + the kernel). + + Note that since only SIGTERM is supported on Windows, this function is + only useful on Unix systems. + """ + if self.has_kernel: + if hasattr(os, "getpgid") and hasattr(os, "killpg"): + try: + pgid = os.getpgid(self.kernel.pid) + os.killpg(pgid, signum) + return + except OSError: + pass + self.kernel.send_signal(signum) + else: + raise RuntimeError("Cannot signal kernel. No kernel is running!") + + def is_alive(self): + """Is the kernel process still running?""" + if self.has_kernel: + if self.kernel.poll() is None: + return True + else: + return False + else: + # we don't have a kernel + return False + +class IPCKernelManager2(KernelManager2): + """Start a kernel on this machine to listen on IPC (filesystem) sockets""" + transport = 'ipc' + + def _ports(self): + if not self.connection_info: + return [] + return [v for (k, v) in self.connection_info.items() + if k.endswith('_port')] + + def cleanup(self): + for port in self._ports(): + ipcfile = "%s-%i" % (self.ip, port) + try: + os.remove(ipcfile) + except (IOError, OSError): + pass + + super(IPCKernelManager2, self).cleanup() + + +def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): + """Start a new kernel, and return its Manager and a blocking client""" + from .client2 import BlockingKernelClient2 + cwd = cwd or os.getcwd() + + km = KernelManager2(kernel_cmd, cwd=cwd) + km.start_kernel() + kc = BlockingKernelClient2(km.connection_info, manager=km) + try: + kc.wait_for_ready(timeout=startup_timeout) + except RuntimeError: + kc.shutdown() + km.finish_shutdown() + km.cleanup() + raise + + return km, kc + +@contextmanager +def run_kernel(kernel_cmd, **kwargs): + """Context manager to create a kernel in a subprocess. + + The kernel is shut down when the context exits. + + Returns + ------- + kernel_client: connected KernelClient instance + """ + km, kc = start_new_kernel(kernel_cmd, **kwargs) + try: + yield kc + finally: + kc.shutdown() + km.finish_shutdown() + km.cleanup() diff --git a/jupyter_client/tests/test_manager2.py b/jupyter_client/tests/test_manager2.py new file mode 100644 index 000000000..e4de3e50a --- /dev/null +++ b/jupyter_client/tests/test_manager2.py @@ -0,0 +1,92 @@ +"""Tests for KernelManager2""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + + +import json +import os + +pjoin = os.path.join +import signal +from subprocess import PIPE +import sys +import time +from unittest import TestCase + +from traitlets.config.loader import Config +from ipykernel.kernelspec import make_ipkernel_cmd +from jupyter_core import paths +from jupyter_client.manager2 import KernelManager2, run_kernel, start_new_kernel +from .utils import test_env, skip_win32 + +TIMEOUT = 30 + +SIGNAL_KERNEL_CMD = [sys.executable, '-m', 'jupyter_client.tests.signalkernel', + '-f', '{connection_file}'] + +class TestKernelManager(TestCase): + def setUp(self): + self.env_patch = test_env() + self.env_patch.start() + + def tearDown(self): + self.env_patch.stop() + + def test_get_connect_info(self): + km = KernelManager2(make_ipkernel_cmd(), os.getcwd()) + km.start_kernel() + try: + self.assertEqual(set(km.connection_info.keys()), { + 'ip', 'transport', + 'hb_port', 'shell_port', 'stdin_port', 'iopub_port', 'control_port', + 'key', 'signature_scheme', + }) + finally: + km.finish_shutdown(timeout=0) + km.cleanup() + + @skip_win32 + def test_signal_kernel_subprocesses(self): + with run_kernel(SIGNAL_KERNEL_CMD, startup_timeout=5) as kc: + def execute(cmd): + reply = kc.execute(cmd, reply=True) + content = reply['content'] + self.assertEqual(content['status'], 'ok') + return content + + N = 5 + for i in range(N): + execute("start") + time.sleep(1) # make sure subprocs stay up + reply = execute('check') + self.assertEqual(reply['user_expressions']['poll'], [None] * N) + + # start a job on the kernel to be interrupted + kc.execute('sleep') + time.sleep(1) # ensure sleep message has been handled before we interrupt + kc.interrupt() + reply = kc.get_shell_msg(TIMEOUT) + content = reply['content'] + self.assertEqual(content['status'], 'ok') + self.assertEqual(content['user_expressions']['interrupted'], True) + # wait up to 5s for subprocesses to handle signal + for i in range(50): + reply = execute('check') + if reply['user_expressions']['poll'] != [-signal.SIGINT] * N: + time.sleep(0.1) + else: + break + # verify that subprocesses were interrupted + self.assertEqual(reply['user_expressions']['poll'], + [-signal.SIGINT] * N) + + def test_start_new_kernel(self): + km, kc = start_new_kernel(make_ipkernel_cmd(), startup_timeout=5) + try: + self.assertTrue(km.is_alive()) + self.assertTrue(kc.is_alive()) + finally: + kc.shutdown() + km.finish_shutdown() + km.cleanup() From fd5f40009f04d878bf0c70ee78340aea1b62253d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 11:20:50 +0000 Subject: [PATCH 32/49] Start recombining launcher interface with manager --- jupyter_client/client2.py | 2 +- jupyter_client/discovery.py | 15 +-- jupyter_client/launcher2.py | 60 +---------- jupyter_client/manager2.py | 132 +++++++++++++++++++------ jupyter_client/tests/test_discovery.py | 18 ++-- jupyter_client/tests/test_manager2.py | 15 +-- 6 files changed, 135 insertions(+), 107 deletions(-) diff --git a/jupyter_client/client2.py b/jupyter_client/client2.py index 02041699e..bd5ca28af 100644 --- a/jupyter_client/client2.py +++ b/jupyter_client/client2.py @@ -363,7 +363,7 @@ def interrupt(self, _header=None): self._send(self.control_socket, msg) return msg['header']['msg_id'] elif self.owned_kernel: - self.manager.interrupt_kernel() + self.manager.interrupt() else: self.log.warning("Can't send signal to non-owned kernel") diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 592dcc77d..ad9bab164 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -4,7 +4,7 @@ import six from .kernelspec import KernelSpecManager -from .launcher2 import PopenKernelLauncher +from .manager2 import KernelManager2 log = logging.getLogger(__name__) @@ -18,7 +18,7 @@ def find_kernels(self): @abstractmethod def launch(self, name, cwd=None): - """Launch a kernel, return an object with the KernelLauncher interface. + """Launch a kernel, return an object with the KernelManager2 interface. name will be one of the kernel names produced by find_kernels() @@ -56,8 +56,9 @@ def find_kernels(self): def launch(self, name, cwd=None): spec = self.ksm.get_kernel_spec(name) - return PopenKernelLauncher(cmd_template=spec.argv, - extra_env=spec.env, cwd=cwd) + km = KernelManager2(kernel_cmd=spec.argv, extra_env=spec.env, cwd=cwd) + km.start_kernel() + return km def launch_async(self, name, cwd=None): from .async_launcher import AsyncPopenKernelLauncher @@ -99,8 +100,10 @@ def launch(self, name, cwd=None): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") - return PopenKernelLauncher(cmd_template=info['spec']['argv'], - extra_env={}, cwd=cwd) + km = KernelManager2(kernel_cmd=info['spec']['argv'], extra_env={}, + cwd=cwd) + km.start_kernel() + return km def launch_async(self, name, cwd=None): from .async_launcher import AsyncPopenKernelLauncher diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py index 7edfbf80b..b08f3b747 100644 --- a/jupyter_client/launcher2.py +++ b/jupyter_client/launcher2.py @@ -1,4 +1,7 @@ -from abc import ABCMeta, abstractmethod +"""Machinery for launching a kernel locally. + +Used by jupyter_client.manager2. +""" from binascii import b2a_hex import errno import json @@ -7,7 +10,7 @@ import six import socket import stat -from subprocess import Popen, PIPE +from subprocess import PIPE import sys import warnings @@ -232,56 +235,3 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): env['JPY_PARENT_PID'] = str(os.getpid()) return kwargs - -class KernelLauncher(six.with_metaclass(ABCMeta, object)): - @abstractmethod - def is_alive(self): - """Check whether the kernel is currently alive (e.g. the process exists) - """ - pass - - @abstractmethod - def wait(self): - """Wait for the kernel process to exit. - """ - raise NotImplementedError() - - @abstractmethod - def signal(self, signum): - """Send a signal to the kernel.""" - pass - - def cleanup(self): - """Clean up any resources.""" - pass - - @abstractmethod - def get_connection_info(self): - """Return a dictionary of connection information""" - pass - -class PopenKernelLauncher(KernelLauncher): - def __init__(self, cmd_template, extra_env=None, cwd=None): - self.cmd_template = cmd_template - self.extra_env = extra_env - self.cwd = cwd - self.connection_file, self.connection_info = make_connection_file() - kwargs = build_popen_kwargs(self.cmd_template, self.connection_file, - self.extra_env, self.cwd) - self.popen = Popen(**kwargs) - - def poll(self): - return self.popen.poll() is None - - def wait(self): - return self.popen.wait() - - def send_signal(self, signum): - self.popen.send_signal(signum) - - def cleanup(self): - if self.connection_file: - os.unlink(self.connection_file) - - def get_connection_info(self): - return self.connection_info diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index 96f0bdbb1..929f9c1c3 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -5,10 +5,12 @@ from __future__ import absolute_import +from abc import ABCMeta, abstractmethod from contextlib import contextmanager import os import signal -from subprocess import Popen +import six +from subprocess import Popen, TimeoutExpired import sys import time @@ -17,7 +19,63 @@ from .launcher2 import make_connection_file, build_popen_kwargs from .localinterfaces import is_local_ip, local_ips, localhost -class KernelManager2(object): + +class KernelManager2ABC(six.with_metaclass(ABCMeta, object)): + @abstractmethod + def is_alive(self): + """Check whether the kernel is currently alive (e.g. the process exists) + """ + pass + + @abstractmethod + def wait(self, timeout): + """Wait for the kernel process to exit. + + If timeout is a number, it is a maximum time in seconds to wait. + timeout=None means wait indefinitely. + + Returns True if the kernel is still alive after waiting, False if it + exited (like is_alive()). + """ + pass + + @abstractmethod + def signal(self, signum): + """Send a signal to the kernel.""" + pass + + @abstractmethod + def interrupt(self): + """Interrupt the kernel by sending it a signal or similar event + + Kernels can request to get interrupts as messages rather than signals. + The manager is *not* expected to handle this. + :meth:`.KernelClient2.interrupt` should send an interrupt_request or + call this method as appropriate. + """ + pass + + @abstractmethod + def kill(self): + """Forcibly terminate the kernel. + + This method may be used to dispose of a kernel that won't shut down. + Working kernels should usually be shut down by sending shutdown_request + from a client and giving it some time to clean up. + """ + pass + + def cleanup(self): + """Clean up any resources, such as files created by the manager.""" + pass + + @abstractmethod + def get_connection_info(self): + """Return a dictionary of connection information""" + pass + + +class KernelManager2(KernelManager2ABC): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen to listen on TCP sockets. @@ -76,22 +134,27 @@ def start_kernel(self): self.log.debug("Starting kernel: %s", kw['args']) self.kernel = Popen(**kw) - def finish_shutdown(self, timeout=5.0, pollinterval=0.1): - """Wait for kernel shutdown, then kill process if it doesn't shutdown. + def wait(self, timeout): + """""" + if timeout is None: + # Wait indefinitely + self.kernel.wait() + return False - This does not send shutdown requests - use :meth:`.KernelClient2.shutdown` - first. - """ - for i in range(int(timeout/pollinterval)): - if self.is_alive(): - time.sleep(pollinterval) - else: - break + if six.PY3: + try: + self.kernel.wait(timeout) + return False + except TimeoutExpired: + return True else: - # OK, we've waited long enough. - if self.has_kernel: - self.log.debug("Kernel is taking too long to finish, killing") - self._kill_kernel() + pollinterval = 0.1 + for i in range(int(timeout / pollinterval)): + if self.is_alive(): + time.sleep(pollinterval) + else: + return False + return self.is_alive() def cleanup(self): """Clean up resources when the kernel is shut down""" @@ -108,13 +171,10 @@ def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None - def _kill_kernel(self): + def kill(self): """Kill the running kernel. - - This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: @@ -138,7 +198,7 @@ def _kill_kernel(self): else: raise RuntimeError("Cannot kill kernel. No kernel is running!") - def interrupt_kernel(self): + def interrupt(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all @@ -153,11 +213,11 @@ def interrupt_kernel(self): from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: - self.signal_kernel(signal.SIGINT) + self.signal(signal.SIGINT) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") - def signal_kernel(self, signum): + def signal(self, signum): """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). @@ -188,6 +248,11 @@ def is_alive(self): # we don't have a kernel return False + def get_connection_info(self): + if self.connection_info is None: + raise RuntimeError("Kernel not started") + return self.connection_info + class IPCKernelManager2(KernelManager2): """Start a kernel on this machine to listen on IPC (filesystem) sockets""" transport = 'ipc' @@ -208,6 +273,19 @@ def cleanup(self): super(IPCKernelManager2, self).cleanup() +def shutdown(client, manager, wait_time=5.0): + """Shutdown a kernel using a client and a manager. + + Attempts a clean shutdown by sending a shutdown message. If the kernel + hasn't exited in wait_time seconds, it will be killed. Set wait_time=None + to wait indefinitely. + """ + client.shutdown() + if manager.wait(wait_time): + # OK, we've waited long enough. + manager.log.debug("Kernel is taking too long to finish, killing") + manager.kill() + manager.cleanup() def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): """Start a new kernel, and return its Manager and a blocking client""" @@ -220,9 +298,7 @@ def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: - kc.shutdown() - km.finish_shutdown() - km.cleanup() + shutdown(kc, km) raise return km, kc @@ -241,6 +317,4 @@ def run_kernel(kernel_cmd, **kwargs): try: yield kc finally: - kc.shutdown() - km.finish_shutdown() - km.cleanup() + shutdown(kc, km) diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index fd5269592..c077cfacf 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -1,7 +1,7 @@ import sys from jupyter_client import discovery -from jupyter_client.launcher2 import KernelLauncher +from jupyter_client.manager2 import KernelManager2ABC def test_ipykernel_provider(): import ipykernel # Fail clearly if ipykernel not installed @@ -21,23 +21,29 @@ def find_kernels(self): yield 'sample', {'argv': ['dummy_kernel']} def launch(self, name, cwd=None): - return DummyKernelLauncher() + return DummyKernelManager() -class DummyKernelLauncher(KernelLauncher): +class DummyKernelManager(KernelManager2ABC): def is_alive(self): """Check whether the kernel is currently alive (e.g. the process exists) """ return True - def wait(self): + def wait(self, timeout): """Wait for the kernel process to exit. """ - return 0 + return False def signal(self, signum): """Send a signal to the kernel.""" pass + def interrupt(self): + pass + + def kill(self): + pass + def get_connection_info(self): """Return a dictionary of connection information""" return {} @@ -48,4 +54,4 @@ def test_meta_kernel_finder(): [('dummy/sample', {'argv': ['dummy_kernel']})] launcher = kf.launch('dummy/sample') - assert isinstance(launcher, DummyKernelLauncher) + assert isinstance(launcher, DummyKernelManager) diff --git a/jupyter_client/tests/test_manager2.py b/jupyter_client/tests/test_manager2.py index e4de3e50a..4c58ca5a7 100644 --- a/jupyter_client/tests/test_manager2.py +++ b/jupyter_client/tests/test_manager2.py @@ -3,21 +3,18 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - -import json import os pjoin = os.path.join import signal -from subprocess import PIPE import sys import time from unittest import TestCase -from traitlets.config.loader import Config from ipykernel.kernelspec import make_ipkernel_cmd -from jupyter_core import paths -from jupyter_client.manager2 import KernelManager2, run_kernel, start_new_kernel +from jupyter_client.manager2 import ( + KernelManager2, run_kernel, start_new_kernel, shutdown +) from .utils import test_env, skip_win32 TIMEOUT = 30 @@ -43,7 +40,7 @@ def test_get_connect_info(self): 'key', 'signature_scheme', }) finally: - km.finish_shutdown(timeout=0) + km.kill() km.cleanup() @skip_win32 @@ -87,6 +84,4 @@ def test_start_new_kernel(self): self.assertTrue(km.is_alive()) self.assertTrue(kc.is_alive()) finally: - kc.shutdown() - km.finish_shutdown() - km.cleanup() + shutdown(kc, km) From 4b935923dbd68da38ba6b542cf716f9f8e3f6483 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 11:24:48 +0000 Subject: [PATCH 33/49] Rename async_launcher -> async_manager --- jupyter_client/{async_launcher.py => async_manager.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jupyter_client/{async_launcher.py => async_manager.py} (100%) diff --git a/jupyter_client/async_launcher.py b/jupyter_client/async_manager.py similarity index 100% rename from jupyter_client/async_launcher.py rename to jupyter_client/async_manager.py From 56b0b49b88fbf961da097891dfd45f3fe773f6e6 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 12:04:01 +0000 Subject: [PATCH 34/49] Bring async manager in line with synchronous one --- jupyter_client/async_manager.py | 155 +++++++++++++-------- jupyter_client/tests/test_async_manager.py | 50 +++++++ 2 files changed, 145 insertions(+), 60 deletions(-) create mode 100644 jupyter_client/tests/test_async_manager.py diff --git a/jupyter_client/async_manager.py b/jupyter_client/async_manager.py index 0dc4df9b6..71d2a81a7 100644 --- a/jupyter_client/async_manager.py +++ b/jupyter_client/async_manager.py @@ -1,84 +1,111 @@ """Launch and control kernels using asyncio. """ -from abc import ABC, abstractmethod # noinspection PyCompatibility import asyncio +import os from .launcher2 import make_connection_file, build_popen_kwargs - -class AsyncKernelLauncher(ABC): - """Interface for async kernel launchers. - - This is very similar to the KernelLauncher interface, but its methods - are asyncio coroutines. There is no poll method, but you can get a future - from the wait method and then poll it by checking ``future.done()``. - """ - @abstractmethod - def wait(self): - """Wait for the kernel process to exit. - """ - raise NotImplementedError() - - @abstractmethod - def send_signal(self, signum): - """Send a signal to the kernel.""" - pass - - def cleanup(self): - """Clean up any resources.""" - pass - - @abstractmethod - def get_connection_info(self): - """Return a dictionary of connection information""" - pass - +from .localinterfaces import is_local_ip, local_ips +from .manager2 import KernelManager2, KernelManager2ABC # noinspection PyCompatibility -class AsyncPopenKernelLauncher(AsyncKernelLauncher): - """Launch a kernel asynchronously in a subprocess. +class AsyncPopenKernelManager(KernelManager2): + """Run a kernel asynchronously in a subprocess. This is the async counterpart to PopenKernelLauncher. """ - process = None - connection_file = None - connection_info = None + _exit_future = None - def __init__(self, process, connection_file, connection_info): - self.process = process - self.connection_file = connection_file - self.connection_info = connection_info + @asyncio.coroutine + def start_kernel(self): + if self.transport == 'tcp' and not is_local_ip(self.ip): + raise RuntimeError("Can only launch a kernel on a local interface. " + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % local_ips() + ) + + self.connection_file, self.connection_info = \ + make_connection_file(self.ip, self.transport) + + kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, + self.extra_env, self.cwd) + + # launch the kernel subprocess + args = kw.pop('args') + self.log.debug("Starting kernel: %s", args) + self.kernel = yield from asyncio.create_subprocess_exec(*args, **kw) + self._exit_future = asyncio.ensure_future(self.kernel.wait()) + + @asyncio.coroutine + def wait(self, timeout): + try: + yield from asyncio.wait_for(self.kernel.wait(), timeout) + return False + except asyncio.TimeoutError: + return True - # __init__ can't be async, so this is the preferred constructor: - @classmethod @asyncio.coroutine - def launch(cls, cmd_template, extra_env, cwd): - connection_file, connection_info = make_connection_file() - kwargs = build_popen_kwargs(cmd_template, connection_file, - extra_env, cwd) - args = kwargs.pop('args') - p = yield from asyncio.create_subprocess_exec(*args, **kwargs) - return cls(p, connection_file, connection_info) + def is_alive(self): + return not (self._exit_future and self._exit_future.done()) @asyncio.coroutine - def wait(self): - return (yield from self.process.wait()) + def signal(self, signum): + return self.kernel.send_signal(signum) @asyncio.coroutine - def send_signal(self, signum): - return self.process.send_signal(signum) + def interrupt(self): + return super().interrupt() + + @asyncio.coroutine + def kill(self): + return self.kernel.kill() @asyncio.coroutine def cleanup(self): - super().cleanup() + return super().cleanup() @asyncio.coroutine def get_connection_info(self): - return self.connection_info + return super().get_connection_info() + +# noinspection PyCompatibility +@asyncio.coroutine +def shutdown(client, manager, wait_time=5.0): + """Shutdown a kernel using a client and a manager. + Attempts a clean shutdown by sending a shutdown message. If the kernel + hasn't exited in wait_time seconds, it will be killed. Set wait_time=None + to wait indefinitely. + """ + client.shutdown() + if (yield from manager.wait(wait_time)): + # OK, we've waited long enough. + manager.log.debug("Kernel is taking too long to finish, killing") + manager.kill() + manager.cleanup() # noinspection PyCompatibility -class AsyncLauncherWrapper(AsyncKernelLauncher): +@asyncio.coroutine +def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): + """Start a new kernel, and return its Manager and a blocking client""" + from .client2 import BlockingKernelClient2 + cwd = cwd or os.getcwd() + + km = AsyncPopenKernelManager(kernel_cmd, cwd=cwd) + yield from km.start_kernel() + # TODO: asyncio client + kc = BlockingKernelClient2(km.connection_info, manager=km) + try: + kc.wait_for_ready(timeout=startup_timeout) + except RuntimeError: + yield from shutdown(kc, km) + raise + + return km, kc + +# noinspection PyCompatibility +class AsyncLauncherWrapper(KernelManager2ABC): """Wrap a blocking KernelLauncher to be used asynchronously. This calls the blocking methods in the event loop's default executor. @@ -91,16 +118,24 @@ def in_default_executor(self, f, *args): return self.loop.run_in_executor(None, f, *args) @asyncio.coroutine - def launch(self): - return (yield from self.in_default_executor(self.wrapped.launch)) + def is_alive(self): + return (yield from self.in_default_executor(self.wrapped.is_alive)) + + @asyncio.coroutine + def wait(self, timeout): + return (yield from self.in_default_executor(self.wrapped.wait, timeout)) + + @asyncio.coroutine + def signal(self, signum): + return (yield from self.in_default_executor(self.wrapped.signal, signum)) @asyncio.coroutine - def wait(self): - return (yield from self.in_default_executor(self.wrapped.wait)) + def interrupt(self): + return (yield from self.in_default_executor(self.wrapped.interrupt)) @asyncio.coroutine - def send_signal(self, signum): - return (yield from self.in_default_executor(self.wrapped.send_signal, signum)) + def kill(self): + return (yield from self.in_default_executor(self.wrapped.kill)) @asyncio.coroutine def cleanup(self): diff --git a/jupyter_client/tests/test_async_manager.py b/jupyter_client/tests/test_async_manager.py new file mode 100644 index 000000000..e958e0f96 --- /dev/null +++ b/jupyter_client/tests/test_async_manager.py @@ -0,0 +1,50 @@ +import os +import pytest +from unittest import TestCase + +asyncio = pytest.importorskip('asyncio') + +from ipykernel.kernelspec import make_ipkernel_cmd +from .utils import test_env, skip_win32 +from jupyter_client.async_manager import ( + AsyncPopenKernelManager, shutdown, start_new_kernel +) + +# noinspection PyCompatibility +class TestKernelManager(TestCase): + def setUp(self): + self.env_patch = test_env() + self.env_patch.start() + + def tearDown(self): + self.env_patch.stop() + + @asyncio.coroutine + def t_get_connect_info(self): + km = AsyncPopenKernelManager(make_ipkernel_cmd(), os.getcwd()) + yield from km.start_kernel() + try: + info = yield from km.get_connection_info() + self.assertEqual(set(info.keys()), { + 'ip', 'transport', + 'hb_port', 'shell_port', 'stdin_port', 'iopub_port', 'control_port', + 'key', 'signature_scheme', + }) + finally: + yield from km.kill() + yield from km.cleanup() + + def test_get_connect_info(self): + asyncio.get_event_loop().run_until_complete(self.t_get_connect_info()) + + @asyncio.coroutine + def t_start_new_kernel(self): + km, kc = yield from start_new_kernel(make_ipkernel_cmd(), startup_timeout=5) + try: + self.assertTrue((yield from km.is_alive())) + self.assertTrue(kc.is_alive()) + finally: + yield from shutdown(kc, km) + + def test_start_new_kernel(self): + asyncio.get_event_loop().run_until_complete(self.t_start_new_kernel()) From 7ca1070db10c3f0be72475a20bbcec0ec2faa72f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 15:07:47 +0000 Subject: [PATCH 35/49] Move ioloop based client to new module, improve --- jupyter_client/client2.py | 207 ++-------------------------- jupyter_client/client2_ioloop.py | 228 +++++++++++++++++++++++++++++++ jupyter_client/util.py | 6 + 3 files changed, 248 insertions(+), 193 deletions(-) create mode 100644 jupyter_client/client2_ioloop.py create mode 100644 jupyter_client/util.py diff --git a/jupyter_client/client2.py b/jupyter_client/client2.py index bd5ca28af..00120733d 100644 --- a/jupyter_client/client2.py +++ b/jupyter_client/client2.py @@ -5,22 +5,18 @@ from __future__ import absolute_import -import atexit -import errno from functools import partial from getpass import getpass from six.moves import input import sys -from threading import Thread import time import zmq -from zmq import ZMQError -from zmq.eventloop import ioloop, zmqstream from ipython_genutils.py3compat import string_types, iteritems from traitlets.log import get_logger as get_app_logger from .channels import major_protocol_version, HBChannel from .session import Session +from .util import inherit_docstring try: monotonic = time.monotonic @@ -56,13 +52,6 @@ def validate_string_dict(dct): if not isinstance(v, string_types): raise ValueError('value %r in dict must be a string' % v) -def inherit_docstring(cls): - def decorator(func): - doc = getattr(cls, func.__name__).__doc__ - func.__doc__ = doc - return func - return decorator - class KernelClient2(): """Communicates with a single kernel on any host via zmq channels. @@ -71,7 +60,7 @@ class KernelClient2(): send the message, they don't wait for a reply. To get results, use e.g. :meth:`get_shell_msg` to fetch messages from the shell channel. """ - hb_channel = None + hb_monitor = None def __init__(self, connection_info, manager=None, use_heartbeat=True): self.connection_info = connection_info @@ -98,6 +87,18 @@ def owned_kernel(self): """True if this client 'owns' the kernel, i.e. started it.""" return self.manager is not None + def close(self): + """Close sockets of this client. + + After calling this, the client can no longer be used. + """ + self.iopub_socket.close() + self.shell_socket.close() + self.stdin_socket.close() + self.control_socket.close() + if self.hb_monitor: + self.hb_monitor.stop() + # flag for whether execute requests should be allowed to call raw_input: allow_stdin = True @@ -709,183 +710,3 @@ def execute_interactive(self, code, silent=False, store_history=True, if timeout is not None: timeout = max(0, deadline - monotonic()) return self._recv_reply(msg_id, timeout=timeout) - - -class IOLoopKernelClient2(KernelClient2): - """Runs a zmq IOLoop to send and receive messages. - - Use ClientInThread to run this in a separate thread alongside your - application. - """ - def __init__(self, **kwargs): - super(IOLoopKernelClient2, self).__init__(**kwargs) - self.ioloop = ioloop.IOLoop.instance() - self.handlers = { - 'iopub': [], - 'shell': [], - 'stdin': [], - } - self.shell_stream = zmqstream.ZMQStream(self.shell_socket, self.ioloop) - self.shell_stream.on_recv(partial(self._handle_recv, 'shell')) - self.iopub_stream = zmqstream.ZMQStream(self.iopub_socket, self.ioloop) - self.iopub_stream.on_recv(partial(self._handle_recv, 'iopub')) - self.stdin_stream = zmqstream.ZMQStream(self.stdin_socket, self.ioloop) - self.stdin_stream.on_recv(partial(self._handle_recv, 'stdin')) - - _inspect = None - - def _handle_recv(self, channel, msg): - """Callback for stream.on_recv. - - Unpacks message, and calls handlers with it. - """ - ident,smsg = self.session.feed_identities(msg) - msg = self.session.deserialize(smsg) - # let client inspect messages - if self._inspect: - self._inspect(msg) - self._call_handlers(channel, msg) - - def _call_handlers(self, channel, msg): - for handler in self.handlers[channel]: - try: - handler() - except Exception as e: - self.log.error("Exception from message handler %r", handler, - exc_info=e) - - def add_handler(self, channel, handler): - self.handlers[channel].append(handler) - - def remove_handler(self, channel, handler): - self.handlers[channel].remove(handler)# - -class ClientInThread(Thread): - client = None - _exiting = False - - def __init__(self, connection_info, manager=None): - super(ClientInThread, self).__init__() - self.daemon = True - self.connection_info = connection_info - self.manager = manager - - @staticmethod - @atexit.register - def _notice_exit(): - ClientInThread._exiting = True - - def run(self): - """Run my loop, ignoring EINTR events in the poller""" - self.client = IOLoopKernelClient2() - while True: - try: - self.client.ioloop.start() - except ZMQError as e: - if e.errno == errno.EINTR: - continue - else: - raise - except Exception: - if self._exiting: - break - else: - raise - else: - break - - @property - def ioloop(self): - if self.client: - return self.client.ioloop - - def stop(self): - """Stop the channel's event loop and join its thread. - - This calls :meth:`~threading.Thread.join` and returns when the thread - terminates. :class:`RuntimeError` will be raised if - :meth:`~threading.Thread.start` is called again. - """ - if self.client is not None: - self.client.ioloop.stop() - self.join() - self.close() - - def close(self): - if self.client is not None: - try: - self.client.ioloop.close(all_fds=True) - except Exception: - pass - - def _call_in_thread(self, function, *args, **kwargs): - return self.ioloop.add_callback(function, *args, **kwargs) - - # Client messaging methods -------------------------------- - # These send as much work as possible to the IO thread, but we generate - # the header in the calling thread so we can return the message ID. - - @inherit_docstring(KernelClient2) - def execute(self, *args, **kwargs): - hdr = self.client.session.msg_header('execute_request') - self._call_in_thread(self.client.execute, *args, _header=hdr, **kwargs) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def complete(self, *args, **kwargs): - hdr = self.client.session.msg_header('complete_request') - self._call_in_thread(self.client.complete, *args, _header=hdr, **kwargs) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def inspect(self, *args, **kwargs): - hdr = self.client.session.msg_header('inspect_request') - self._call_in_thread(self.client.inspect, *args, _header=hdr, **kwargs) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def history(self, *args, **kwargs): - hdr = self.client.session.msg_header('history_request') - self._call_in_thread(self.client.history, *args, _header=hdr, **kwargs) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def kernel_info(self, _header=None): - hdr = self.client.session.msg_header('kernel_info_request') - self._call_in_thread(self.client.kernel_info, _header=hdr) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def comm_info(self, target_name=None, _header=None): - hdr = self.client.session.msg_header('comm_info_request') - self._call_in_thread(self.client.comm_info, target_name, _header=hdr) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def shutdown(self, restart=False, _header=None): - hdr = self.client.session.msg_header('shutdown_request') - self._call_in_thread(self.client.shutdown, restart, _header=hdr) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def is_complete(self, code, _header=None): - hdr = self.client.session.msg_header('is_complete_request') - self._call_in_thread(self.client.is_complete, code, _header=hdr) - return hdr['msg_id'] - - @inherit_docstring(KernelClient2) - def interrupt(self, _header=None): - mode = self.connection_info.get('interrupt_mode', 'signal') - if mode == 'message': - hdr = self.client.session.msg_header('is_complete_request') - self._call_in_thread(self.client.interrupt, _header=hdr) - return hdr['msg_id'] - else: - self.client.interrupt() - - @inherit_docstring(KernelClient2) - def input(self, string, parent=None): - hdr = self.client.session.msg_header('input_reply') - self._call_in_thread(self.client.is_complete, string, - parent=parent, _header=hdr) - return hdr['msg_id'] diff --git a/jupyter_client/client2_ioloop.py b/jupyter_client/client2_ioloop.py new file mode 100644 index 000000000..f6bfca8d9 --- /dev/null +++ b/jupyter_client/client2_ioloop.py @@ -0,0 +1,228 @@ +import atexit +import errno +from functools import partial +from threading import Thread +from zmq import ioloop, zmqstream, ZMQError + +from .client2 import KernelClient2 +from .util import inherit_docstring + + +class IOLoopKernelClient2(KernelClient2): + """Uses a zmq/tornado IOLoop to handle received messages and fire callbacks. + + Use ClientInThread to run this in a separate thread alongside your + application. + """ + def __init__(self, **kwargs): + super(IOLoopKernelClient2, self).__init__(**kwargs) + self.ioloop = ioloop.IOLoop.current() + self.handlers = { + 'iopub': [], + 'shell': [self._auto_adapt], + 'stdin': [], + 'control': [], + } + self.shell_stream = zmqstream.ZMQStream(self.shell_socket, self.ioloop) + self.shell_stream.on_recv(partial(self._handle_recv, 'shell')) + self.iopub_stream = zmqstream.ZMQStream(self.iopub_socket, self.ioloop) + self.iopub_stream.on_recv(partial(self._handle_recv, 'iopub')) + self.stdin_stream = zmqstream.ZMQStream(self.stdin_socket, self.ioloop) + self.stdin_stream.on_recv(partial(self._handle_recv, 'stdin')) + self.control_stream = zmqstream.ZMQStream(self.control_socket, self.ioloop) + self.control_stream.on_recv(partial(self._handle_recv, 'control')) + + def close(self): + """Close the client's sockets & streams. + + This does not close the IOLoop. + """ + self.shell_stream.close() + self.iopub_stream.close() + self.stdin_stream.close() + self.control_stream.close() + if self.hb_monitor: + self.hb_monitor.stop() + + def _auto_adapt(self, msg): + """Use the first kernel_info_reply to set up protocol version adaptation + """ + if msg['header']['msg_type'] == 'kernel_info_reply': + self._handle_kernel_info_reply(msg) + self.remove_handler('shell', self._auto_adapt) + + def _handle_recv(self, channel, msg): + """Callback for stream.on_recv. + + Unpacks message, and calls handlers with it. + """ + ident, smsg = self.session.feed_identities(msg) + msg = self.session.deserialize(smsg) + self._call_handlers(channel, msg) + + def _call_handlers(self, channel, msg): + for handler in self.handlers[channel]: + try: + handler(msg) + except Exception as e: + self.log.error("Exception from message handler %r", handler, + exc_info=e) + + def add_handler(self, channel, handler): + """Add a callback for received messages on one channel. + + Parameters + ---------- + + channel : str + One of 'shell', 'iopub', 'stdin' or 'control' + handler : function + Will be called for each message received with the message dictionary + as a single argument. + """ + self.handlers[channel].append(handler) + + def remove_handler(self, channel, handler): + """Remove a previously registered callback.""" + self.handlers[channel].remove(handler) + +class ClientInThread(Thread): + """Run an IOLoopKernelClient2 in a separate thread. + + The main client methods (execute, complete, etc.) all pass their arguments + to the ioloop thread, which sends the messages. Handlers for received + messages will be called in the ioloop thread, so they should typically + use a signal or callback mechanism to interact with the application in + the main thread. + """ + client = None + _exiting = False + + def __init__(self, connection_info, manager=None): + super(ClientInThread, self).__init__() + self.daemon = True + self.connection_info = connection_info + self.manager = manager + + @staticmethod + @atexit.register + def _notice_exit(): + ClientInThread._exiting = True + + def run(self): + """Run my loop, ignoring EINTR events in the poller""" + self.client = IOLoopKernelClient2() + try: + self._run_loop() + finally: + self.client.close() + self.client.ioloop.close() + self.client = None + + def _run_loop(self): + while True: + try: + self.client.ioloop.start() + except ZMQError as e: + if e.errno == errno.EINTR: + continue + else: + raise + except Exception: + if self._exiting: + break + else: + raise + else: + break + + @property + def ioloop(self): + if self.client: + return self.client.ioloop + + def close(self): + """Shut down the client and wait for the thread to exit. + + This closes the client's sockets and ioloop, and joins its thread. + """ + if self.client is not None: + self.ioloop.add_callback(self.client.ioloop.stop) + self.join() + + @inherit_docstring(IOLoopKernelClient2) + def add_handler(self, channel, handler): + self.client.handlers[channel].append(handler) + + @inherit_docstring(IOLoopKernelClient2) + def remove_handler(self, channel, handler): + self.client.handlers[channel].remove(handler) + + # Client messaging methods -------------------------------- + # These send as much work as possible to the IO thread, but we generate + # the header in the calling thread so we can return the message ID. + + @inherit_docstring(KernelClient2) + def execute(self, *args, **kwargs): + hdr = self.client.session.msg_header('execute_request') + self.ioloop.add_callback(self.client.execute, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def complete(self, *args, **kwargs): + hdr = self.client.session.msg_header('complete_request') + self.ioloop.add_callback(self.client.complete, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def inspect(self, *args, **kwargs): + hdr = self.client.session.msg_header('inspect_request') + self.ioloop.add_callback(self.client.inspect, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def history(self, *args, **kwargs): + hdr = self.client.session.msg_header('history_request') + self.ioloop.add_callback(self.client.history, *args, _header=hdr, **kwargs) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def kernel_info(self, _header=None): + hdr = self.client.session.msg_header('kernel_info_request') + self.ioloop.add_callback(self.client.kernel_info, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def comm_info(self, target_name=None, _header=None): + hdr = self.client.session.msg_header('comm_info_request') + self.ioloop.add_callback(self.client.comm_info, target_name, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def shutdown(self, restart=False, _header=None): + hdr = self.client.session.msg_header('shutdown_request') + self.ioloop.add_callback(self.client.shutdown, restart, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def is_complete(self, code, _header=None): + hdr = self.client.session.msg_header('is_complete_request') + self.ioloop.add_callback(self.client.is_complete, code, _header=hdr) + return hdr['msg_id'] + + @inherit_docstring(KernelClient2) + def interrupt(self, _header=None): + mode = self.connection_info.get('interrupt_mode', 'signal') + if mode == 'message': + hdr = self.client.session.msg_header('is_complete_request') + self.ioloop.add_callback(self.client.interrupt, _header=hdr) + return hdr['msg_id'] + else: + self.client.interrupt() + + @inherit_docstring(KernelClient2) + def input(self, string, parent=None): + hdr = self.client.session.msg_header('input_reply') + self.ioloop.add_callback(self.client.is_complete, string, + parent=parent, _header=hdr) + return hdr['msg_id'] diff --git a/jupyter_client/util.py b/jupyter_client/util.py new file mode 100644 index 000000000..4e4681f24 --- /dev/null +++ b/jupyter_client/util.py @@ -0,0 +1,6 @@ +def inherit_docstring(cls): + def decorator(func): + doc = getattr(cls, func.__name__).__doc__ + func.__doc__ = doc + return func + return decorator From 64b47a69c99b610d05e60b70cae214586838629f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 15:50:22 +0000 Subject: [PATCH 36/49] Managers start kernel on creation --- jupyter_client/async_manager.py | 64 ++++++---- jupyter_client/discovery.py | 22 ++-- jupyter_client/manager2.py | 136 +++++++-------------- jupyter_client/tests/test_async_manager.py | 3 +- jupyter_client/tests/test_manager2.py | 1 - 5 files changed, 95 insertions(+), 131 deletions(-) diff --git a/jupyter_client/async_manager.py b/jupyter_client/async_manager.py index 71d2a81a7..7e6ab4de4 100644 --- a/jupyter_client/async_manager.py +++ b/jupyter_client/async_manager.py @@ -4,8 +4,10 @@ import asyncio import os +from traitlets.log import get_logger as get_app_logger + from .launcher2 import make_connection_file, build_popen_kwargs -from .localinterfaces import is_local_ip, local_ips +from .localinterfaces import is_local_ip, local_ips, localhost from .manager2 import KernelManager2, KernelManager2ABC # noinspection PyCompatibility @@ -13,29 +15,42 @@ class AsyncPopenKernelManager(KernelManager2): """Run a kernel asynchronously in a subprocess. This is the async counterpart to PopenKernelLauncher. + Use the launch() class method to make an instance, because the constructor + can't be async. """ _exit_future = None + def __init__(self, kernel, connection_info, connection_file): + self.kernel = kernel + self.connection_info = connection_info + self.connection_file = connection_file + self.log = get_app_logger() + self._exit_future = asyncio.ensure_future(self.kernel.wait()) + + @classmethod @asyncio.coroutine - def start_kernel(self): - if self.transport == 'tcp' and not is_local_ip(self.ip): + def launch(cls, kernel_cmd, cwd, extra_env=None, ip=None): + if ip is None: + ip = localhost() + + if cls.transport == 'tcp' and not is_local_ip(ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % local_ips() ) - self.connection_file, self.connection_info = \ - make_connection_file(self.ip, self.transport) + connection_file, connection_info = \ + make_connection_file(ip, cls.transport) - kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, - self.extra_env, self.cwd) + kw = build_popen_kwargs(kernel_cmd, connection_file, + extra_env, cwd) # launch the kernel subprocess args = kw.pop('args') - self.log.debug("Starting kernel: %s", args) - self.kernel = yield from asyncio.create_subprocess_exec(*args, **kw) - self._exit_future = asyncio.ensure_future(self.kernel.wait()) + get_app_logger().debug("Starting kernel: %s", args) + kernel = yield from asyncio.create_subprocess_exec(*args, **kw) + return cls(kernel, connection_info, connection_file) @asyncio.coroutine def wait(self, timeout): @@ -47,11 +62,11 @@ def wait(self, timeout): @asyncio.coroutine def is_alive(self): - return not (self._exit_future and self._exit_future.done()) + return not self._exit_future.done() @asyncio.coroutine def signal(self, signum): - return self.kernel.send_signal(signum) + return super().signal(signum) @asyncio.coroutine def interrupt(self): @@ -59,7 +74,7 @@ def interrupt(self): @asyncio.coroutine def kill(self): - return self.kernel.kill() + return super().kill() @asyncio.coroutine def cleanup(self): @@ -67,7 +82,7 @@ def cleanup(self): @asyncio.coroutine def get_connection_info(self): - return super().get_connection_info() + return self.connection_info # noinspection PyCompatibility @asyncio.coroutine @@ -92,8 +107,7 @@ def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): from .client2 import BlockingKernelClient2 cwd = cwd or os.getcwd() - km = AsyncPopenKernelManager(kernel_cmd, cwd=cwd) - yield from km.start_kernel() + km = yield from AsyncPopenKernelManager.launch(kernel_cmd, cwd=cwd) # TODO: asyncio client kc = BlockingKernelClient2(km.connection_info, manager=km) try: @@ -105,7 +119,7 @@ def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): return km, kc # noinspection PyCompatibility -class AsyncLauncherWrapper(KernelManager2ABC): +class AsyncManagerWrapper(KernelManager2ABC): """Wrap a blocking KernelLauncher to be used asynchronously. This calls the blocking methods in the event loop's default executor. @@ -114,33 +128,33 @@ def __init__(self, wrapped, loop=None): self.wrapped = wrapped self.loop = loop or asyncio.get_event_loop() - def in_default_executor(self, f, *args): + def _exec(self, f, *args): return self.loop.run_in_executor(None, f, *args) @asyncio.coroutine def is_alive(self): - return (yield from self.in_default_executor(self.wrapped.is_alive)) + return (yield from self._exec(self.wrapped.is_alive)) @asyncio.coroutine def wait(self, timeout): - return (yield from self.in_default_executor(self.wrapped.wait, timeout)) + return (yield from self._exec(self.wrapped.wait, timeout)) @asyncio.coroutine def signal(self, signum): - return (yield from self.in_default_executor(self.wrapped.signal, signum)) + return (yield from self._exec(self.wrapped.signal, signum)) @asyncio.coroutine def interrupt(self): - return (yield from self.in_default_executor(self.wrapped.interrupt)) + return (yield from self._exec(self.wrapped.interrupt)) @asyncio.coroutine def kill(self): - return (yield from self.in_default_executor(self.wrapped.kill)) + return (yield from self._exec(self.wrapped.kill)) @asyncio.coroutine def cleanup(self): - return (yield from self.in_default_executor(self.wrapped.cleanup)) + return (yield from self._exec(self.wrapped.cleanup)) @asyncio.coroutine def get_connection_info(self): - return (yield from self.in_default_executor(self.wrapped.get_connection_info)) + return (yield from self._exec(self.wrapped.get_connection_info)) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index ad9bab164..18d70188d 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -56,15 +56,13 @@ def find_kernels(self): def launch(self, name, cwd=None): spec = self.ksm.get_kernel_spec(name) - km = KernelManager2(kernel_cmd=spec.argv, extra_env=spec.env, cwd=cwd) - km.start_kernel() - return km + return KernelManager2(kernel_cmd=spec.argv, extra_env=spec.env, cwd=cwd) def launch_async(self, name, cwd=None): - from .async_launcher import AsyncPopenKernelLauncher + from .async_manager import AsyncPopenKernelManager spec = self.ksm.get_kernel_spec(name) - return AsyncPopenKernelLauncher.launch(cmd_template=spec.argv, - extra_env=spec.env, cwd=cwd) + return AsyncPopenKernelManager.launch( + kernel_cmd=spec.argv, extra_env=spec.env, cwd=cwd) class IPykernelProvider(KernelProviderBase): """Offers a kernel type using the Python interpreter it's running in. @@ -100,18 +98,16 @@ def launch(self, name, cwd=None): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") - km = KernelManager2(kernel_cmd=info['spec']['argv'], extra_env={}, - cwd=cwd) - km.start_kernel() - return km + return KernelManager2(kernel_cmd=info['spec']['argv'], extra_env={}, + cwd=cwd) def launch_async(self, name, cwd=None): - from .async_launcher import AsyncPopenKernelLauncher + from .async_manager import AsyncPopenKernelManager info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") - return AsyncPopenKernelLauncher.launch( - cmd_template=info['spec']['argv'], extra_env={}, cwd=cwd) + return AsyncPopenKernelManager.launch( + kernel_cmd=info['spec']['argv'], extra_env={}, cwd=cwd) class KernelFinder(object): """Manages a collection of kernel providers to find available kernel types diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index 929f9c1c3..da0290812 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -97,27 +97,12 @@ class KernelManager2(KernelManager2ABC): """ transport = 'tcp' - # The kernel process with which the KernelManager is communicating. - # generally a Popen instance - kernel = None - - # The dictionary of info to connect to the kernel, and the file storing it - connection_info = None - connection_file = None - def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): - self.kernel_cmd = kernel_cmd - self.cwd = cwd - self.extra_env = extra_env if ip is None: ip = localhost() - self.ip = ip self.log = get_app_logger() - def start_kernel(self): - """Starts a kernel on this host in a separate process. - """ - if self.transport == 'tcp' and not is_local_ip(self.ip): + if self.transport == 'tcp' and not is_local_ip(ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " @@ -125,10 +110,10 @@ def start_kernel(self): ) self.connection_file, self.connection_info = \ - make_connection_file(self.ip, self.transport) + make_connection_file(ip, self.transport) - kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, - self.extra_env, self.cwd) + kw = build_popen_kwargs(kernel_cmd, self.connection_file, + extra_env, cwd) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kw['args']) @@ -158,45 +143,34 @@ def wait(self, timeout): def cleanup(self): """Clean up resources when the kernel is shut down""" - if self.connection_file: - # cleanup connection files on full shutdown of kernel we started - try: - os.remove(self.connection_file) - except (IOError, OSError, AttributeError): - pass - self.connection_file = None - - @property - def has_kernel(self): - """Has a kernel been started that we are managing.""" - return self.kernel is not None + # cleanup connection files on full shutdown of kernel we started + try: + os.remove(self.connection_file) + except (IOError, OSError, AttributeError): + pass def kill(self): """Kill the running kernel. """ - if self.has_kernel: - # Signal the kernel to terminate (sends SIGKILL on Unix and calls - # TerminateProcess() on Win32). - try: - self.kernel.kill() - except OSError as e: - # In Windows, we will get an Access Denied error if the process - # has already terminated. Ignore it. - if sys.platform == 'win32': - if e.winerror != 5: - raise - # On Unix, we may get an ESRCH error if the process has already - # terminated. Ignore it. - else: - from errno import ESRCH - if e.errno != ESRCH: - raise + # Signal the kernel to terminate (sends SIGKILL on Unix and calls + # TerminateProcess() on Win32). + try: + self.kernel.kill() + except OSError as e: + # In Windows, we will get an Access Denied error if the process + # has already terminated. Ignore it. + if sys.platform == 'win32': + if e.winerror != 5: + raise + # On Unix, we may get an ESRCH error if the process has already + # terminated. Ignore it. + else: + from errno import ESRCH + if e.errno != ESRCH: + raise - # Block until the kernel terminates. - self.kernel.wait() - self.kernel = None - else: - raise RuntimeError("Cannot kill kernel. No kernel is running!") + # Block until the kernel terminates. + self.kernel.wait() def interrupt(self): """Interrupts the kernel by sending it a signal. @@ -208,14 +182,11 @@ def interrupt(self): a signal. This method does *not* handle that. Use KernelClient.interrupt to send a message or a signal as appropriate. """ - if self.has_kernel: - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - self.signal(signal.SIGINT) + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) else: - raise RuntimeError("Cannot interrupt kernel. No kernel is running!") + self.signal(signal.SIGINT) def signal(self, signum): """Sends a signal to the process group of the kernel (this @@ -225,47 +196,31 @@ def signal(self, signum): Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ - if self.has_kernel: - if hasattr(os, "getpgid") and hasattr(os, "killpg"): - try: - pgid = os.getpgid(self.kernel.pid) - os.killpg(pgid, signum) - return - except OSError: - pass - self.kernel.send_signal(signum) - else: - raise RuntimeError("Cannot signal kernel. No kernel is running!") + if hasattr(os, "getpgid") and hasattr(os, "killpg"): + try: + pgid = os.getpgid(self.kernel.pid) + os.killpg(pgid, signum) + return + except OSError: + pass + self.kernel.send_signal(signum) def is_alive(self): """Is the kernel process still running?""" - if self.has_kernel: - if self.kernel.poll() is None: - return True - else: - return False - else: - # we don't have a kernel - return False + return self.kernel.poll() is None def get_connection_info(self): - if self.connection_info is None: - raise RuntimeError("Kernel not started") return self.connection_info class IPCKernelManager2(KernelManager2): """Start a kernel on this machine to listen on IPC (filesystem) sockets""" transport = 'ipc' - def _ports(self): - if not self.connection_info: - return [] - return [v for (k, v) in self.connection_info.items() - if k.endswith('_port')] - def cleanup(self): - for port in self._ports(): - ipcfile = "%s-%i" % (self.ip, port) + ports = [v for (k, v) in self.connection_info.items() + if k.endswith('_port')] + for port in ports: + ipcfile = "%s-%i" % (self.connection_info['ip'], port) try: os.remove(ipcfile) except (IOError, OSError): @@ -293,12 +248,12 @@ def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): cwd = cwd or os.getcwd() km = KernelManager2(kernel_cmd, cwd=cwd) - km.start_kernel() kc = BlockingKernelClient2(km.connection_info, manager=km) try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: shutdown(kc, km) + kc.close() raise return km, kc @@ -318,3 +273,4 @@ def run_kernel(kernel_cmd, **kwargs): yield kc finally: shutdown(kc, km) + kc.close() diff --git a/jupyter_client/tests/test_async_manager.py b/jupyter_client/tests/test_async_manager.py index e958e0f96..446a76ba7 100644 --- a/jupyter_client/tests/test_async_manager.py +++ b/jupyter_client/tests/test_async_manager.py @@ -21,8 +21,7 @@ def tearDown(self): @asyncio.coroutine def t_get_connect_info(self): - km = AsyncPopenKernelManager(make_ipkernel_cmd(), os.getcwd()) - yield from km.start_kernel() + km = yield from AsyncPopenKernelManager.launch(make_ipkernel_cmd(), os.getcwd()) try: info = yield from km.get_connection_info() self.assertEqual(set(info.keys()), { diff --git a/jupyter_client/tests/test_manager2.py b/jupyter_client/tests/test_manager2.py index 4c58ca5a7..cf38b1ec0 100644 --- a/jupyter_client/tests/test_manager2.py +++ b/jupyter_client/tests/test_manager2.py @@ -32,7 +32,6 @@ def tearDown(self): def test_get_connect_info(self): km = KernelManager2(make_ipkernel_cmd(), os.getcwd()) - km.start_kernel() try: self.assertEqual(set(km.connection_info.keys()), { 'ip', 'transport', From 673dda7406bf0f3ec91dffe5ddb4f3524773dea0 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 15:54:01 +0000 Subject: [PATCH 37/49] Clarify docstring --- jupyter_client/discovery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 18d70188d..1adde5a5a 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -31,8 +31,9 @@ def launch_async(self, name, cwd=None): name will be one of the kernel names produced by find_kernels() - This method should return an asyncio future, which resolves to an object - with the AsyncKernelLauncher interface. + This method should act as an asyncio coroutine, returning an object + with the AsyncKernelManager interface. This closely matches the + synchronous KernelManager2 interface, but all methods are coroutines. """ raise NotImplementedError() From 5c70b5683445acb881555a7c28c9eb8b45aba708 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 16:05:34 +0000 Subject: [PATCH 38/49] Fix use of interrupt event for Windows, close stdin pipe --- jupyter_client/async_manager.py | 41 +++++++++++++++++++++++++++++---- jupyter_client/launcher2.py | 19 ++++++++------- jupyter_client/manager2.py | 8 +++++-- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/jupyter_client/async_manager.py b/jupyter_client/async_manager.py index 7e6ab4de4..05ed723d7 100644 --- a/jupyter_client/async_manager.py +++ b/jupyter_client/async_manager.py @@ -6,9 +6,12 @@ from traitlets.log import get_logger as get_app_logger -from .launcher2 import make_connection_file, build_popen_kwargs +from .launcher2 import ( + make_connection_file, build_popen_kwargs, prepare_interrupt_event +) from .localinterfaces import is_local_ip, local_ips, localhost from .manager2 import KernelManager2, KernelManager2ABC +from .util import inherit_docstring # noinspection PyCompatibility class AsyncPopenKernelManager(KernelManager2): @@ -20,16 +23,35 @@ class AsyncPopenKernelManager(KernelManager2): """ _exit_future = None - def __init__(self, kernel, connection_info, connection_file): + def __init__(self, kernel, connection_info, connection_file, + win_interrupt_evt): self.kernel = kernel self.connection_info = connection_info self.connection_file = connection_file + self._win_interrupt_evt = win_interrupt_evt self.log = get_app_logger() self._exit_future = asyncio.ensure_future(self.kernel.wait()) @classmethod @asyncio.coroutine def launch(cls, kernel_cmd, cwd, extra_env=None, ip=None): + """Main constructor for async kernel manager + + Parameters + ---------- + + kernel_cmd : list of str + The Popen command template to launch the kernel + cwd : str + The working directory to launch the kernel in + extra_env : dict, optional + Dictionary of environment variables to update the existing environment + ip : str, optional + Set the kernel\'s IP address [default localhost]. + If the IP address is something other than localhost, then + Consoles on other machines will be able to connect + to the Kernel, so be careful! + """ if ip is None: ip = localhost() @@ -43,15 +65,18 @@ def launch(cls, kernel_cmd, cwd, extra_env=None, ip=None): connection_file, connection_info = \ make_connection_file(ip, cls.transport) - kw = build_popen_kwargs(kernel_cmd, connection_file, - extra_env, cwd) + kw = build_popen_kwargs(kernel_cmd, connection_file, extra_env, cwd) + win_interrupt_evt = prepare_interrupt_event(kw['env']) # launch the kernel subprocess args = kw.pop('args') get_app_logger().debug("Starting kernel: %s", args) kernel = yield from asyncio.create_subprocess_exec(*args, **kw) - return cls(kernel, connection_info, connection_file) + kernel.stdin.close() + return cls(kernel, connection_info, connection_file, win_interrupt_evt) + + @inherit_docstring(KernelManager2) @asyncio.coroutine def wait(self, timeout): try: @@ -60,26 +85,32 @@ def wait(self, timeout): except asyncio.TimeoutError: return True + @inherit_docstring(KernelManager2) @asyncio.coroutine def is_alive(self): return not self._exit_future.done() + @inherit_docstring(KernelManager2) @asyncio.coroutine def signal(self, signum): return super().signal(signum) + @inherit_docstring(KernelManager2) @asyncio.coroutine def interrupt(self): return super().interrupt() + @inherit_docstring(KernelManager2) @asyncio.coroutine def kill(self): return super().kill() + @inherit_docstring(KernelManager2) @asyncio.coroutine def cleanup(self): return super().cleanup() + @inherit_docstring(KernelManager2) @asyncio.coroutine def get_connection_info(self): return self.connection_info diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py index b08f3b747..8800ef246 100644 --- a/jupyter_client/launcher2.py +++ b/jupyter_client/launcher2.py @@ -193,14 +193,6 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): kwargs['cwd'] = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii') - from .win_interrupt import create_interrupt_event - # Create a Win32 event for interrupting the kernel - # and store it in an environment variable. - interrupt_event = create_interrupt_event() - env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) - # deprecated old env name: - env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] - try: # noinspection PyUnresolvedReferences from _winapi import DuplicateHandle, GetCurrentProcess, \ @@ -235,3 +227,14 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): env['JPY_PARENT_PID'] = str(os.getpid()) return kwargs + +def prepare_interrupt_event(env): + if sys.platform == 'win32': + from .win_interrupt import create_interrupt_event + # Create a Win32 event for interrupting the kernel + # and store it in an environment variable. + interrupt_event = create_interrupt_event() + env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) + # deprecated old env name: + env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] + return interrupt_event diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index da0290812..956b6c59d 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -16,7 +16,9 @@ from traitlets.log import get_logger as get_app_logger -from .launcher2 import make_connection_file, build_popen_kwargs +from .launcher2 import ( + make_connection_file, build_popen_kwargs, prepare_interrupt_event +) from .localinterfaces import is_local_ip, local_ips, localhost @@ -114,10 +116,12 @@ def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): kw = build_popen_kwargs(kernel_cmd, self.connection_file, extra_env, cwd) + self._win_interrupt_evt = prepare_interrupt_event(kw['env']) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kw['args']) self.kernel = Popen(**kw) + self.kernel.stdin.close() def wait(self, timeout): """""" @@ -184,7 +188,7 @@ def interrupt(self): """ if sys.platform == 'win32': from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) + send_interrupt(self._win_interrupt_evt) else: self.signal(signal.SIGINT) From 910020c84ed3592934b6aa2d884897d3395d9f61 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 16:26:15 +0000 Subject: [PATCH 39/49] No TimeoutExpired on Py2 --- jupyter_client/manager2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index 956b6c59d..aa9153491 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -10,7 +10,7 @@ import os import signal import six -from subprocess import Popen, TimeoutExpired +import subprocess import sys import time @@ -120,7 +120,7 @@ def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): # launch the kernel subprocess self.log.debug("Starting kernel: %s", kw['args']) - self.kernel = Popen(**kw) + self.kernel = subprocess.Popen(**kw) self.kernel.stdin.close() def wait(self, timeout): @@ -134,7 +134,7 @@ def wait(self, timeout): try: self.kernel.wait(timeout) return False - except TimeoutExpired: + except subprocess.TimeoutExpired: return True else: pollinterval = 0.1 From 9c6f55f806a1d62ef366e750b51537a773a3ed84 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 16:27:03 +0000 Subject: [PATCH 40/49] Drop Python 3.3 tests from Travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a3a96915..8c372062c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ language: python python: - "nightly" - - '3.6-dev' + - 3.6 - 3.5 - 3.4 - - 3.3 - 2.7 sudo: false install: From 626b83282cc95c7a0a406bdf880c210410bfcd75 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 16:43:06 +0000 Subject: [PATCH 41/49] Misc Python 2 fixes --- jupyter_client/client2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/client2.py b/jupyter_client/client2.py index 00120733d..067e9ac91 100644 --- a/jupyter_client/client2.py +++ b/jupyter_client/client2.py @@ -3,7 +3,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from __future__ import absolute_import +from __future__ import absolute_import, print_function from functools import partial from getpass import getpass @@ -52,7 +52,7 @@ def validate_string_dict(dct): if not isinstance(v, string_types): raise ValueError('value %r in dict must be a string' % v) -class KernelClient2(): +class KernelClient2(object): """Communicates with a single kernel on any host via zmq channels. The messages that can be sent are exposed as methods of the From bce030ccd53ad1baee984730040eef31c6f48fb0 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 5 Dec 2017 16:50:38 +0000 Subject: [PATCH 42/49] Ignore asyncio tests on Python 2 --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8c372062c..820c3d19c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,12 @@ install: - pip install --upgrade setuptools pip - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov script: - - py.test --cov jupyter_client jupyter_client + - | + if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + py.test --cov jupyter_client jupyter_client --ignore jupyter_client/tests/test_async_manager.py + else + py.test --cov jupyter_client jupyter_client + fi after_success: - codecov matrix: From a550bb43bc8d4dcd2c77a21488164917e944ff57 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 6 Dec 2017 15:07:44 +0000 Subject: [PATCH 43/49] Fix getting messages from correct socket --- jupyter_client/client2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/client2.py b/jupyter_client/client2.py index 067e9ac91..da798ecf3 100644 --- a/jupyter_client/client2.py +++ b/jupyter_client/client2.py @@ -436,9 +436,9 @@ def _get_msg(self, socket, block=True, timeout=None): if block: if timeout is not None: timeout *= 1000 # seconds to ms - ready = self.shell_socket.poll(timeout) + ready = socket.poll(timeout) else: - ready = self.shell_socket.poll(timeout=0) + ready = socket.poll(timeout=0) if ready: return self._recv(socket) From eebeca4f5780070162c79d8d1c5a4ff70edfc45e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 6 Dec 2017 15:39:06 +0000 Subject: [PATCH 44/49] Copy client tests for KernelClient2 --- jupyter_client/tests/test_client2.py | 86 ++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 jupyter_client/tests/test_client2.py diff --git a/jupyter_client/tests/test_client2.py b/jupyter_client/tests/test_client2.py new file mode 100644 index 000000000..3cab7a2e8 --- /dev/null +++ b/jupyter_client/tests/test_client2.py @@ -0,0 +1,86 @@ +"""Tests for the KernelClient2""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + + +import os + +pjoin = os.path.join +from unittest import TestCase + +from ipykernel.kernelspec import make_ipkernel_cmd +from ..manager2 import start_new_kernel, shutdown +from .utils import test_env + +from ipython_genutils.py3compat import string_types +from IPython.utils.capture import capture_output + +TIMEOUT = 30 + + +class TestKernelClient(TestCase): + def setUp(self): + self.env_patch = test_env() + self.env_patch.start() + self.addCleanup(self.env_patch.stop) + + self.km, self.kc = start_new_kernel(kernel_cmd=make_ipkernel_cmd()) + self.addCleanup(self.kc.close) + self.addCleanup(shutdown, self.kc, self.km) + + def test_execute_interactive(self): + kc = self.kc + + with capture_output() as io: + reply = kc.execute_interactive("print('hello')", timeout=TIMEOUT) + assert 'hello' in io.stdout + assert reply['content']['status'] == 'ok' + + def _check_reply(self, reply_type, reply): + self.assertIsInstance(reply, dict) + self.assertEqual(reply['header']['msg_type'], reply_type + '_reply') + self.assertEqual(reply['parent_header']['msg_type'], + reply_type + '_request') + + def test_history(self): + kc = self.kc + msg_id = kc.history(session=0) + self.assertIsInstance(msg_id, string_types) + reply = kc.history(session=0, reply=True, timeout=TIMEOUT) + self._check_reply('history', reply) + + def test_inspect(self): + kc = self.kc + msg_id = kc.inspect('who cares') + self.assertIsInstance(msg_id, string_types) + reply = kc.inspect('code', reply=True, timeout=TIMEOUT) + self._check_reply('inspect', reply) + + def test_complete(self): + kc = self.kc + msg_id = kc.complete('who cares') + self.assertIsInstance(msg_id, string_types) + reply = kc.complete('code', reply=True, timeout=TIMEOUT) + self._check_reply('complete', reply) + + def test_kernel_info(self): + kc = self.kc + msg_id = kc.kernel_info() + self.assertIsInstance(msg_id, string_types) + reply = kc.kernel_info(reply=True, timeout=TIMEOUT) + self._check_reply('kernel_info', reply) + + def test_comm_info(self): + kc = self.kc + msg_id = kc.comm_info() + self.assertIsInstance(msg_id, string_types) + reply = kc.comm_info(reply=True, timeout=TIMEOUT) + self._check_reply('comm_info', reply) + + def test_shutdown(self): + kc = self.kc + msg_id = kc.shutdown() + self.assertIsInstance(msg_id, string_types) + reply = kc.shutdown(reply=True, timeout=TIMEOUT) + self._check_reply('shutdown', reply) From 07ba2ea59f4ba965564a1038ff7c136e27f59659 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 6 Dec 2017 16:37:56 +0000 Subject: [PATCH 45/49] Add tests for running a KernelClient in a separate thread --- jupyter_client/client2_ioloop.py | 20 +++-- jupyter_client/tests/test_client2_loop.py | 95 +++++++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 jupyter_client/tests/test_client2_loop.py diff --git a/jupyter_client/client2_ioloop.py b/jupyter_client/client2_ioloop.py index f6bfca8d9..5e42045c0 100644 --- a/jupyter_client/client2_ioloop.py +++ b/jupyter_client/client2_ioloop.py @@ -1,8 +1,9 @@ import atexit import errno from functools import partial -from threading import Thread -from zmq import ioloop, zmqstream, ZMQError +from threading import Thread, Event +from zmq import ZMQError +from zmq.eventloop import ioloop, zmqstream from .client2 import KernelClient2 from .util import inherit_docstring @@ -14,8 +15,8 @@ class IOLoopKernelClient2(KernelClient2): Use ClientInThread to run this in a separate thread alongside your application. """ - def __init__(self, **kwargs): - super(IOLoopKernelClient2, self).__init__(**kwargs) + def __init__(self, connection_info, **kwargs): + super(IOLoopKernelClient2, self).__init__(connection_info, **kwargs) self.ioloop = ioloop.IOLoop.current() self.handlers = { 'iopub': [], @@ -61,7 +62,9 @@ def _handle_recv(self, channel, msg): self._call_handlers(channel, msg) def _call_handlers(self, channel, msg): - for handler in self.handlers[channel]: + # [:] copies the list - handlers that remove themselves (or add other + # handlers) will not mess up iterating over it. + for handler in self.handlers[channel][:]: try: handler(msg) except Exception as e: @@ -98,11 +101,12 @@ class ClientInThread(Thread): client = None _exiting = False - def __init__(self, connection_info, manager=None): + def __init__(self, connection_info, manager=None, loop=None): super(ClientInThread, self).__init__() self.daemon = True self.connection_info = connection_info self.manager = manager + self.started = Event() @staticmethod @atexit.register @@ -111,7 +115,9 @@ def _notice_exit(): def run(self): """Run my loop, ignoring EINTR events in the poller""" - self.client = IOLoopKernelClient2() + loop = ioloop.IOLoop(make_current=True) + self.client = IOLoopKernelClient2(self.connection_info, manager=self.manager) + self.client.ioloop.add_callback(self.started.set) try: self._run_loop() finally: diff --git a/jupyter_client/tests/test_client2_loop.py b/jupyter_client/tests/test_client2_loop.py new file mode 100644 index 000000000..72ea8bd64 --- /dev/null +++ b/jupyter_client/tests/test_client2_loop.py @@ -0,0 +1,95 @@ +"""Tests for the ioloop KernelClient running in a separate thread.""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + + +import os +try: + from queue import Queue, Empty +except ImportError: + from Queue import Queue, Empty + +pjoin = os.path.join +from unittest import TestCase + +from ipykernel.kernelspec import make_ipkernel_cmd +from jupyter_client.manager2 import KernelManager2, shutdown +from jupyter_client.client2_ioloop import ClientInThread +from .utils import test_env + +from ipython_genutils.py3compat import string_types +from IPython.utils.capture import capture_output + +TIMEOUT = 30 + + +class TestKernelClient(TestCase): + def setUp(self): + self.env_patch = test_env() + self.env_patch.start() + self.addCleanup(self.env_patch.stop) + + # Start a client in a new thread, put received messages in queues. + self.km = KernelManager2(make_ipkernel_cmd(), cwd='.') + self.kc = ClientInThread(self.km.get_connection_info(), manager=self.km) + self.received = {'shell': Queue(), 'iopub': Queue()} + self.kc.start() + if not self.kc.started.wait(10.0): + raise RuntimeError("Failed to start kernel client") + self.kc.add_handler('shell', self.received['shell'].put) + self.kc.add_handler('iopub', self.received['iopub'].put) + + + def tearDown(self): + shutdown(self.kc, self.km) + self.kc.close() + self.env_patch.stop() + + def _check_reply(self, reply_type, reply): + self.assertIsInstance(reply, dict) + self.assertEqual(reply['header']['msg_type'], reply_type + '_reply') + self.assertEqual(reply['parent_header']['msg_type'], + reply_type + '_request') + + def test_history(self): + kc = self.kc + msg_id = kc.history(session=0) + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('history', reply) + + def test_inspect(self): + kc = self.kc + msg_id = kc.inspect('who cares') + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('inspect', reply) + + def test_complete(self): + kc = self.kc + msg_id = kc.complete('who cares') + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('complete', reply) + + def test_kernel_info(self): + kc = self.kc + msg_id = kc.kernel_info() + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('kernel_info', reply) + + def test_comm_info(self): + kc = self.kc + msg_id = kc.comm_info() + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('comm_info', reply) + + def test_shutdown(self): + kc = self.kc + msg_id = kc.shutdown() + self.assertIsInstance(msg_id, string_types) + reply = self.received['shell'].get(timeout=TIMEOUT) + self._check_reply('shutdown', reply) From f9d8493dda3592ad45c67cc4451b86a32fe18962 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 14:29:02 +0000 Subject: [PATCH 46/49] Add relaunch() method to kernel manager classes --- jupyter_client/async_manager.py | 97 ++++++++++++---------- jupyter_client/launcher2.py | 5 +- jupyter_client/manager2.py | 33 ++++++++ jupyter_client/tests/test_async_manager.py | 3 +- jupyter_client/tests/test_discovery.py | 3 + 5 files changed, 96 insertions(+), 45 deletions(-) diff --git a/jupyter_client/async_manager.py b/jupyter_client/async_manager.py index 05ed723d7..19f27b350 100644 --- a/jupyter_client/async_manager.py +++ b/jupyter_client/async_manager.py @@ -18,63 +18,61 @@ class AsyncPopenKernelManager(KernelManager2): """Run a kernel asynchronously in a subprocess. This is the async counterpart to PopenKernelLauncher. - Use the launch() class method to make an instance, because the constructor - can't be async. + After instantiating, call the launch() method to actually launch the + subprocess. This is necessary because the constructor cannot be async. + + Parameters + ---------- + + kernel_cmd : list of str + The Popen command template to launch the kernel + cwd : str + The working directory to launch the kernel in + extra_env : dict, optional + Dictionary of environment variables to update the existing environment + ip : str, optional + Set the kernel\'s IP address [default localhost]. + If the IP address is something other than localhost, then + Consoles on other machines will be able to connect + to the Kernel, so be careful! """ _exit_future = None + _win_interrupt_evt = None + kernel = None - def __init__(self, kernel, connection_info, connection_file, - win_interrupt_evt): - self.kernel = kernel - self.connection_info = connection_info - self.connection_file = connection_file - self._win_interrupt_evt = win_interrupt_evt - self.log = get_app_logger() - self._exit_future = asyncio.ensure_future(self.kernel.wait()) - - @classmethod - @asyncio.coroutine - def launch(cls, kernel_cmd, cwd, extra_env=None, ip=None): - """Main constructor for async kernel manager - - Parameters - ---------- - - kernel_cmd : list of str - The Popen command template to launch the kernel - cwd : str - The working directory to launch the kernel in - extra_env : dict, optional - Dictionary of environment variables to update the existing environment - ip : str, optional - Set the kernel\'s IP address [default localhost]. - If the IP address is something other than localhost, then - Consoles on other machines will be able to connect - to the Kernel, so be careful! - """ + def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): + self.kernel_cmd = kernel_cmd + self.cwd = cwd + self.extra_env = extra_env if ip is None: ip = localhost() + self.ip = ip + self.log = get_app_logger() - if cls.transport == 'tcp' and not is_local_ip(ip): + if self.transport == 'tcp' and not is_local_ip(ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % local_ips() ) - connection_file, connection_info = \ - make_connection_file(ip, cls.transport) + self.connection_file, self.connection_info = \ + make_connection_file(ip, self.transport) - kw = build_popen_kwargs(kernel_cmd, connection_file, extra_env, cwd) - win_interrupt_evt = prepare_interrupt_event(kw['env']) + @asyncio.coroutine + def launch(self): + """Run this immediately after instantiation to launch the kernel process. + """ + kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, + self.extra_env, self.cwd) + self._win_interrupt_evt = prepare_interrupt_event(kw['env']) # launch the kernel subprocess args = kw.pop('args') - get_app_logger().debug("Starting kernel: %s", args) - kernel = yield from asyncio.create_subprocess_exec(*args, **kw) - kernel.stdin.close() - - return cls(kernel, connection_info, connection_file, win_interrupt_evt) + self.log.debug("Starting kernel: %s", args) + self.kernel = yield from asyncio.create_subprocess_exec(*args, **kw) + self.kernel.stdin.close() + self._exit_future = asyncio.ensure_future(self.kernel.wait()) @inherit_docstring(KernelManager2) @asyncio.coroutine @@ -115,6 +113,20 @@ def cleanup(self): def get_connection_info(self): return self.connection_info + @inherit_docstring(KernelManager2) + @asyncio.coroutine + def relaunch(self): + kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, + self.extra_env, self.cwd) + prepare_interrupt_event(kw['env'], self._win_interrupt_evt) + + # launch the kernel subprocess + args = kw.pop('args') + self.log.debug("Starting kernel: %s", args) + self.kernel = yield from asyncio.create_subprocess_exec(*args, **kw) + self.kernel.stdin.close() + self._exit_future = asyncio.ensure_future(self.kernel.wait()) + # noinspection PyCompatibility @asyncio.coroutine def shutdown(client, manager, wait_time=5.0): @@ -138,7 +150,8 @@ def start_new_kernel(kernel_cmd, startup_timeout=60, cwd=None): from .client2 import BlockingKernelClient2 cwd = cwd or os.getcwd() - km = yield from AsyncPopenKernelManager.launch(kernel_cmd, cwd=cwd) + km = AsyncPopenKernelManager(kernel_cmd, cwd=cwd) + yield from km.launch() # TODO: asyncio client kc = BlockingKernelClient2(km.connection_info, manager=km) try: diff --git a/jupyter_client/launcher2.py b/jupyter_client/launcher2.py index 8800ef246..a05cc2486 100644 --- a/jupyter_client/launcher2.py +++ b/jupyter_client/launcher2.py @@ -228,12 +228,13 @@ def build_popen_kwargs(cmd_template, connection_file, extra_env=None, cwd=None): return kwargs -def prepare_interrupt_event(env): +def prepare_interrupt_event(env, interrupt_event=None): if sys.platform == 'win32': from .win_interrupt import create_interrupt_event # Create a Win32 event for interrupting the kernel # and store it in an environment variable. - interrupt_event = create_interrupt_event() + if interrupt_event is None: + interrupt_event = create_interrupt_event() env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) # deprecated old env name: env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index aa9153491..45b4f87ef 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -76,6 +76,17 @@ def get_connection_info(self): """Return a dictionary of connection information""" pass + @abstractmethod + def relaunch(self): + """Attempt to relaunch the kernel using the same ports. + + This is meant to be called after the managed kernel has died. Calling + it while the kernel is still alive has undefined behaviour. + + Returns True if this manager supports that. + """ + pass + class KernelManager2(KernelManager2ABC): """Manages a single kernel in a subprocess on this host. @@ -100,8 +111,12 @@ class KernelManager2(KernelManager2ABC): transport = 'tcp' def __init__(self, kernel_cmd, cwd, extra_env=None, ip=None): + self.kernel_cmd = kernel_cmd + self.cwd = cwd + self.extra_env = extra_env if ip is None: ip = localhost() + self.ip = ip self.log = get_app_logger() if self.transport == 'tcp' and not is_local_ip(ip): @@ -153,6 +168,24 @@ def cleanup(self): except (IOError, OSError, AttributeError): pass + def relaunch(self): + """Attempt to relaunch the kernel using the same ports. + + This is meant to be called after the managed kernel has died. Calling + it while the kernel is still alive has undefined behaviour. + + Returns True if this manager supports that. + """ + kw = build_popen_kwargs(self.kernel_cmd, self.connection_file, + self.extra_env, self.cwd) + prepare_interrupt_event(kw['env'], self._win_interrupt_evt) + + # launch the kernel subprocess + self.log.debug("Starting kernel: %s", kw['args']) + self.kernel = subprocess.Popen(**kw) + self.kernel.stdin.close() + return True + def kill(self): """Kill the running kernel. """ diff --git a/jupyter_client/tests/test_async_manager.py b/jupyter_client/tests/test_async_manager.py index 446a76ba7..c90b45adc 100644 --- a/jupyter_client/tests/test_async_manager.py +++ b/jupyter_client/tests/test_async_manager.py @@ -21,7 +21,8 @@ def tearDown(self): @asyncio.coroutine def t_get_connect_info(self): - km = yield from AsyncPopenKernelManager.launch(make_ipkernel_cmd(), os.getcwd()) + km = AsyncPopenKernelManager(make_ipkernel_cmd(), os.getcwd()) + yield from km.launch() try: info = yield from km.get_connection_info() self.assertEqual(set(info.keys()), { diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index c077cfacf..1f7c6fdda 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -48,6 +48,9 @@ def get_connection_info(self): """Return a dictionary of connection information""" return {} + def relaunch(self): + return True + def test_meta_kernel_finder(): kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) assert list(kf.find_kernels()) == \ From 8c6d5eea24f724b0ae9d7a268938bdc722ae29c8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 15:11:47 +0000 Subject: [PATCH 47/49] Add new restarter machinery --- jupyter_client/restarter2.py | 149 ++++++++++++++++++++++++ jupyter_client/tests/test_discovery.py | 5 +- jupyter_client/tests/test_restarter2.py | 37 ++++++ 3 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 jupyter_client/restarter2.py create mode 100644 jupyter_client/tests/test_restarter2.py diff --git a/jupyter_client/restarter2.py b/jupyter_client/restarter2.py new file mode 100644 index 000000000..ab7c4221b --- /dev/null +++ b/jupyter_client/restarter2.py @@ -0,0 +1,149 @@ +"""Machinery to monitor a KernelManager and restart the kernel if it dies +""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from tornado import ioloop +from traitlets.config.configurable import LoggingConfigurable +from traitlets import ( + Float, Bool, Integer, +) + +from .discovery import KernelFinder + + +class KernelRestarterBase(LoggingConfigurable): + """Monitor and autorestart a kernel.""" + + debug = Bool(False, config=True, + help="""Whether to include every poll event in debugging output. + + Has to be set explicitly, because there will be *a lot* of output. + """ + ) + + time_to_dead = Float(3.0, config=True, + help="""Kernel heartbeat interval in seconds.""" + ) + + restart_limit = Integer(5, config=True, + help="""The number of consecutive autorestarts before the kernel is presumed dead.""" + ) + + random_ports_until_alive = Bool(True, config=True, + help="""Whether to choose new random ports when restarting before the kernel is alive.""" + ) + _restarting = False + _restart_count = 0 + _initial_startup = True + + def __init__(self, kernel_manager, kernel_type, kernel_finder=None, **kw): + super(KernelRestarterBase, self).__init__(**kw) + self.kernel_manager = kernel_manager + self.kernel_type = kernel_type + self.kernel_finder = kernel_finder or KernelFinder.from_entrypoints() + self.callbacks = dict(restart=[], dead=[]) + + def start(self): + """Start monitoring the kernel.""" + raise NotImplementedError("Must be implemented in a subclass") + + def stop(self): + """Stop monitoring.""" + raise NotImplementedError("Must be implemented in a subclass") + + def add_callback(self, f, event='restart'): + """register a callback to fire on a particular event + + Possible values for event: + + 'restart' (default): kernel has died, and will be restarted. + 'dead': restart has failed, kernel will be left dead. + + """ + self.callbacks[event].append(f) + + def remove_callback(self, f, event='restart'): + """unregister a callback to fire on a particular event + + Possible values for event: + + 'restart' (default): kernel has died, and will be restarted. + 'dead': restart has failed, kernel will be left dead. + + """ + try: + self.callbacks[event].remove(f) + except ValueError: + pass + + def _fire_callbacks(self, event): + """fire our callbacks for a particular event""" + for callback in self.callbacks[event]: + try: + callback() + except Exception as e: + self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True) + + def do_restart(self): + """Called when the kernel has died""" + if self._restarting: + self._restart_count += 1 + else: + self._restart_count = 1 + + if self._restart_count >= self.restart_limit: + self.log.warning("KernelRestarter: restart failed") + self._fire_callbacks('dead') + self._restarting = False + self._restart_count = 0 + self.stop() + else: + newports = self.random_ports_until_alive and self._initial_startup + self._fire_callbacks('restart') + if newports: + cwd = getattr(self.kernel_manager, 'cwd', None) # :-/ + self.log.info("KernelRestarter: starting new manager (%i/%i)", + self._restart_count, self.restart_limit) + self.kernel_manager.cleanup() + self.kernel_manager = self.kernel_finder.launch( + self.kernel_type, cwd) + else: + self.log.info( + 'KernelRestarter: restarting kernel (%i/%i), keeping ports', + self._restart_count, self.restart_limit) + self.kernel_manager.relaunch() + self._restarting = True + + def poll(self): + if self.debug: + self.log.debug('Polling kernel...') + if not self.kernel_manager.is_alive(): + self.do_restart() + else: + if self._initial_startup: + self._initial_startup = False + if self._restarting: + self.log.debug("KernelRestarter: restart apparently succeeded") + self._restarting = False + + +class TornadoKernelRestarter(KernelRestarterBase): + """Monitor a kernel using the tornado ioloop.""" + _pcallback = None + + def start(self): + """Start the polling of the kernel.""" + if self._pcallback is None: + self._pcallback = ioloop.PeriodicCallback( + self.poll, 1000*self.time_to_dead, + ) + self._pcallback.start() + + def stop(self): + """Stop the kernel polling.""" + if self._pcallback is not None: + self._pcallback.stop() + self._pcallback = None + diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index 1f7c6fdda..58d680a30 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -24,10 +24,11 @@ def launch(self, name, cwd=None): return DummyKernelManager() class DummyKernelManager(KernelManager2ABC): + _alive = True def is_alive(self): """Check whether the kernel is currently alive (e.g. the process exists) """ - return True + return self._alive def wait(self, timeout): """Wait for the kernel process to exit. @@ -42,7 +43,7 @@ def interrupt(self): pass def kill(self): - pass + self._alive = False def get_connection_info(self): """Return a dictionary of connection information""" diff --git a/jupyter_client/tests/test_restarter2.py b/jupyter_client/tests/test_restarter2.py new file mode 100644 index 000000000..2a8527d63 --- /dev/null +++ b/jupyter_client/tests/test_restarter2.py @@ -0,0 +1,37 @@ +from .test_discovery import DummyKernelManager, DummyKernelProvider + +from jupyter_client import discovery, restarter2 + +def test_reinstantiate(): + # If the kernel fails before the first poll, a new manager should be + # instantiated + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + manager = kf.launch('dummy/sample') + manager.kill() + + restarter = restarter2.KernelRestarterBase(manager, 'dummy/sample', + kernel_finder=kf) + assert restarter.kernel_manager is manager + restarter.poll() + assert restarter.kernel_manager is not manager + assert restarter.kernel_manager.is_alive() + +def test_relaunch(): + # If the kernel fails after the first poll, its manager's relaunch() method + # should be called. + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + manager = kf.launch('dummy/sample') + relaunch_count = [0] + def relaunch(): + relaunch_count[0] += 1 + manager.relaunch = relaunch + + restarter = restarter2.KernelRestarterBase(manager, 'dummy/sample', + kernel_finder=kf) + restarter.poll() + assert relaunch_count[0] == 0 + # Kernel dies after first poll + manager.kill() + restarter.poll() + assert relaunch_count[0] == 1 + assert restarter.kernel_manager is manager From 01e88a84726953e467de39f043170941fbcb27f5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 13 Dec 2017 11:36:06 +0000 Subject: [PATCH 48/49] Update 'jupyter kernel' command to use new APIs --- jupyter_client/discovery.py | 2 ++ jupyter_client/kernelapp.py | 29 ++++++++++++++--------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 1adde5a5a..f37bca8c7 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -155,6 +155,7 @@ def launch(self, name, cwd=None): for provider in self.providers: if provider_id == provider.id: return provider.launch(kernel_id, cwd) + raise KeyError(provider_id) def launch_async(self, name, cwd=None): """Launch a kernel of a given kernel type, using asyncio. @@ -163,3 +164,4 @@ def launch_async(self, name, cwd=None): for provider in self.providers: if provider_id == provider.id: return provider.launch_async(kernel_id, cwd) + raise KeyError(provider_id) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index a2ab17812..ee8167cc3 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -1,14 +1,14 @@ import os import signal -import uuid from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode from . import __version__ -from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME -from .manager import KernelManager +from .discovery import KernelFinder +from .client2 import BlockingKernelClient2 +from .manager2 import shutdown class KernelApp(JupyterApp): """Launch a kernel by name in a local subprocess. @@ -16,24 +16,21 @@ class KernelApp(JupyterApp): version = __version__ description = "Run a kernel locally in a subprocess" - classes = [KernelManager, KernelSpecManager] - aliases = { 'kernel': 'KernelApp.kernel_name', - 'ip': 'KernelManager.ip', + #'ip': 'KernelManager.ip', # TODO } flags = {'debug': base_flags['debug']} - kernel_name = Unicode(NATIVE_KERNEL_NAME, + kernel_name = Unicode("pyimport/kernel", help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): super(KernelApp, self).initialize(argv) - self.km = KernelManager(kernel_name=self.kernel_name, - config=self.config) - cf_basename = 'kernel-%s.json' % uuid.uuid4() - self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.kernel_finder = KernelFinder.from_entrypoints() + if '/' not in self.kernel_name: + self.kernel_name = 'spec/' + self.kernel_name self.loop = IOLoop.current() self.loop.add_callback(self._record_started) @@ -49,11 +46,13 @@ def shutdown_handler(signo, frame): def shutdown(self, signo): self.log.info('Shutting down on signal %d' % signo) - self.km.shutdown_kernel() + client = BlockingKernelClient2(self.manager.get_connection_info()) + shutdown(client, self.manager) + client.close() self.loop.stop() def log_connection_info(self): - cf = self.km.connection_file + cf = self.manager.connection_file self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) @@ -69,13 +68,13 @@ def _record_started(self): def start(self): self.log.info('Starting kernel %r', self.kernel_name) + self.manager = self.kernel_finder.launch(self.kernel_name) try: - self.km.start_kernel() self.log_connection_info() self.setup_signals() self.loop.start() finally: - self.km.cleanup() + self.manager.cleanup() main = KernelApp.launch_instance From ca78b7e039a2c36992db0e66d02f930ade57541d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 13 Feb 2018 17:59:42 +0000 Subject: [PATCH 49/49] Don't rely on KernelManager having a log attribute --- jupyter_client/manager2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter_client/manager2.py b/jupyter_client/manager2.py index 45b4f87ef..81f9012ec 100644 --- a/jupyter_client/manager2.py +++ b/jupyter_client/manager2.py @@ -7,6 +7,7 @@ from abc import ABCMeta, abstractmethod from contextlib import contextmanager +import logging import os import signal import six @@ -14,6 +15,8 @@ import sys import time +log = logging.getLogger(__name__) + from traitlets.log import get_logger as get_app_logger from .launcher2 import ( @@ -275,7 +278,7 @@ def shutdown(client, manager, wait_time=5.0): client.shutdown() if manager.wait(wait_time): # OK, we've waited long enough. - manager.log.debug("Kernel is taking too long to finish, killing") + log.debug("Kernel is taking too long to finish, killing") manager.kill() manager.cleanup()