From cd300a801dfdae7fe0d55e0855873213e64035f2 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Wed, 1 Dec 2021 00:20:14 +0530 Subject: [PATCH 01/19] Added support for Web sockets based server over PYLSP. Using 'Autobahn' with 'twisted' to provied Web Sockets --- pylsp/__main__.py | 9 ++++- pylsp/python_lsp.py | 93 ++++++++++++++++++++++++++++++++++++++++++--- setup.py | 2 + 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/pylsp/__main__.py b/pylsp/__main__.py index 4698d5c9..becb0023 100644 --- a/pylsp/__main__.py +++ b/pylsp/__main__.py @@ -13,7 +13,7 @@ import json from .python_lsp import (PythonLSPServer, start_io_lang_server, - start_tcp_lang_server) + start_tcp_lang_server, start_ws_lang_server) from ._version import __version__ LOG_FORMAT = "%(asctime)s {0} - %(levelname)s - %(name)s - %(message)s".format( @@ -27,6 +27,10 @@ def add_arguments(parser): "--tcp", action="store_true", help="Use TCP server instead of stdio" ) + parser.add_argument( + "--ws", action="store_true", + help="Use Web Sockets server instead of stdio" + ) parser.add_argument( "--host", default="127.0.0.1", help="Bind to this address" @@ -72,6 +76,9 @@ def main(): if args.tcp: start_tcp_lang_server(args.host, args.port, args.check_parent_process, PythonLSPServer) + elif args.ws: + start_ws_lang_server(args.host, args.port, args.check_parent_process, + PythonLSPServer) else: stdin, stdout = _binary_stdio() start_io_lang_server(stdin, stdout, args.check_parent_process, diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index c71f08b3..518d3328 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -91,6 +91,66 @@ def start_io_lang_server(rfile, wfile, check_parent_process, handler_class): server.start() +def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): + if not issubclass(handler_class, PythonLSPServer): + raise ValueError('Handler class must be an instance of PythonLSPServer') + + # imports needed only for websocket based server + from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory + from twisted.internet import reactor + import ujson as json + + class MyServerProtocol(WebSocketServerProtocol): + + def handler(self, message): + """Handler to send responses of processed requests to respective web socket clients""" + try: + # we have to send bytes to the sendmessage method, So encoding to utf8 + payload = json.dumps(message, ensure_ascii = False).encode('utf8') + + # Binary payload support will be provided in future + self.sendMessage(payload, isBinary=False) + except Exception as e: # pylint: disable=broad-except + log.exception("Failed to write message to output file %s, %s", payload, str(e)) + + def onConnect(self, request): + log.debug(f"new WS connection opened for {request.peer}") + + def onOpen(self): + log.debug("Creating LSP object") + # Not using default stream reader and writer. + # Instead using a consumer based approach to handle processed requests + self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, check_parent_process=check_parent_process) + + def onMessage(self, payload, isBinary): + if isBinary: + # binary message support will be provided in future + # print(f"Binary message received: {len(payload)} bytes") + pass + else: + # we get payload as byte array and so decoding it to string using utf8 + log.debug("consuming payload and feeding it LSP handler") + self._lsp.consume(json.loads(payload.decode('utf8'))) + + def onClose(self, wasClean, code, reason): + log.debug("shutting down and exiting LSP handler") + self._lsp.m_shutdown() + self._lsp.m_exit() + log.debug(f"WS connection closed due to : {reason}!") + + factory = WebSocketServerFactory() + factory.protocol = MyServerProtocol + # factory.setProtocolOptions(maxConnections=2) + + # This only supports IPv4 connections + reactor.listenTCP(port, factory) + + log.info(f"Starting Web Sockets server on port: {port}") + + # runs forever + reactor.run() + + class PythonLSPServer(MethodDispatcher): """ Implementation of the Microsoft VSCode Language Server Protocol https://github.com/Microsoft/language-server-protocol/blob/master/versions/protocol-1-x.md @@ -98,7 +158,7 @@ class PythonLSPServer(MethodDispatcher): # pylint: disable=too-many-public-methods,redefined-builtin - def __init__(self, rx, tx, check_parent_process=False): + def __init__(self, rx, tx, check_parent_process=False, consumer=None): self.workspace = None self.config = None self.root_uri = None @@ -106,16 +166,35 @@ def __init__(self, rx, tx, check_parent_process=False): self.workspaces = {} self.uri_workspace_mapper = {} - self._jsonrpc_stream_reader = JsonRpcStreamReader(rx) - self._jsonrpc_stream_writer = JsonRpcStreamWriter(tx) self._check_parent_process = check_parent_process - self._endpoint = Endpoint(self, self._jsonrpc_stream_writer.write, max_workers=MAX_WORKERS) + + if rx is not None: + self._jsonrpc_stream_reader = JsonRpcStreamReader(rx) + else: + self._jsonrpc_stream_reader = None + + if tx is not None: + self._jsonrpc_stream_writer = JsonRpcStreamWriter(tx) + else: + self._jsonrpc_stream_writer = None + + # if Consumer is None, It is assumed that the default streams based approach is being used + if consumer is None: + self._endpoint = Endpoint(self, self._jsonrpc_stream_writer.write, max_workers=MAX_WORKERS) + else: + self._endpoint = Endpoint(self, consumer, max_workers=MAX_WORKERS) + self._dispatchers = [] self._shutdown = False def start(self): """Entry point for the server.""" self._jsonrpc_stream_reader.listen(self._endpoint.consume) + + def consume(self, message): + """Entry point for consumer based server. Alternate to stream listeners""" + # assuming message will be JSON + self._endpoint.consume(message) def __getitem__(self, item): """Override getitem to fallback through multiple dispatchers.""" @@ -141,8 +220,10 @@ def m_shutdown(self, **_kwargs): def m_exit(self, **_kwargs): self._endpoint.shutdown() - self._jsonrpc_stream_reader.close() - self._jsonrpc_stream_writer.close() + if self._jsonrpc_stream_reader is not None: + self._jsonrpc_stream_reader.close() + if self._jsonrpc_stream_writer is not None: + self._jsonrpc_stream_writer.close() def _match_uri_to_workspace(self, uri): workspace_uri = _utils.match_uri_to_workspace(uri, self.workspaces) diff --git a/setup.py b/setup.py index f8deee52..142bb165 100755 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ def get_version(module='pylsp'): 'pylint>=2.5.0', 'rope>=0.10.5', 'yapf', + 'autobahn[twisted,accelerate]>=21.11.1' ], 'autopep8': ['autopep8>=1.6.0,<1.7.0'], 'flake8': ['flake8>=4.0.0,<4.1.0'], @@ -65,6 +66,7 @@ def get_version(module='pylsp'): 'pylint': ['pylint>=2.5.0'], 'rope': ['rope>0.10.5'], 'yapf': ['yapf'], + 'autobahn-twisted': ['autobahn[twisted,accelerate]>=21.11.1'], 'test': ['pylint>=2.5.0', 'pytest', 'pytest-cov', 'coverage', 'numpy', 'pandas', 'matplotlib', 'pyqt5', 'flaky'], }, From 565aa727ee1dffea217f6d4fdda3b7af6ebe0ce1 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Wed, 1 Dec 2021 00:33:37 +0530 Subject: [PATCH 02/19] Updated readme with ws configuration usage --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 6b3cec14..56f2119f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,27 @@ If the respective dependencies are found, the following optional providers will - [autopep8](https://github.com/hhatto/autopep8) for code formatting - [YAPF](https://github.com/google/yapf) for code formatting (preferred over autopep8) +PYLSP can now be used in Web Sockets based configuration as follows: + +``` +pylsp --ws --host [host/ip] --port [port] +``` + +The following libraries are required for Web Sockets support: +- [autobahn-twisted] (https://github.com/crossbario/autobahn-python) for PYLSP Web sockets using autobahn in twisted configuration. refer (https://autobahn.readthedocs.io/en/latest/installation.html) for more details + +you can install these dependencies with below command: + +``` +pip install autobahn[twisted,accelerate]>=21.11.1 +``` + +or with this: + +``` +pip install 'python-lsp-server[autobahn-twisted]' +``` + Optional providers can be installed using the `extras` syntax. To install [YAPF](https://github.com/google/yapf) formatting for example: ``` From 3cb9d24a5d87a4f6ced09ce28b68a2a8dc96d6cf Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Wed, 1 Dec 2021 00:34:27 +0530 Subject: [PATCH 03/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 56f2119f..02105c4d 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ pylsp --ws --host [host/ip] --port [port] ``` The following libraries are required for Web Sockets support: -- [autobahn-twisted] (https://github.com/crossbario/autobahn-python) for PYLSP Web sockets using autobahn in twisted configuration. refer (https://autobahn.readthedocs.io/en/latest/installation.html) for more details +- [autobahn-twisted](https://github.com/crossbario/autobahn-python) for PYLSP Web sockets using autobahn in twisted configuration. refer [Autobahn installation](https://autobahn.readthedocs.io/en/latest/installation.html) for more details you can install these dependencies with below command: From 268cb342126ac1589043d1c0d68e036627c80cc0 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Thu, 2 Dec 2021 00:46:57 +0530 Subject: [PATCH 04/19] Updated pylint checks --- pylsp/python_lsp.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 518d3328..06744564 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -95,11 +95,15 @@ def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): if not issubclass(handler_class, PythonLSPServer): raise ValueError('Handler class must be an instance of PythonLSPServer') + #pylint: disable=wrong-import-position + # imports needed only for websocket based server from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory from twisted.internet import reactor import ujson as json - + + #pylint: enable=wrong-import-position + class MyServerProtocol(WebSocketServerProtocol): def handler(self, message): @@ -113,14 +117,12 @@ def handler(self, message): except Exception as e: # pylint: disable=broad-except log.exception("Failed to write message to output file %s, %s", payload, str(e)) - def onConnect(self, request): - log.debug(f"new WS connection opened for {request.peer}") - def onOpen(self): log.debug("Creating LSP object") # Not using default stream reader and writer. # Instead using a consumer based approach to handle processed requests - self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, check_parent_process=check_parent_process) + self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, + check_parent_process=check_parent_process) # pylint: disable=attribute-defined-outside-init def onMessage(self, payload, isBinary): if isBinary: @@ -132,15 +134,15 @@ def onMessage(self, payload, isBinary): log.debug("consuming payload and feeding it LSP handler") self._lsp.consume(json.loads(payload.decode('utf8'))) - def onClose(self, wasClean, code, reason): + def onClose(self, reason): log.debug("shutting down and exiting LSP handler") self._lsp.m_shutdown() self._lsp.m_exit() log.debug(f"WS connection closed due to : {reason}!") - factory = WebSocketServerFactory() + factory = WebSocketServerFactory(f"ws://{bind_addr}:{port}") factory.protocol = MyServerProtocol - # factory.setProtocolOptions(maxConnections=2) + # factory.setProtocolOptions(maxConnections=2) # This only supports IPv4 connections reactor.listenTCP(port, factory) From eef347174a15a46136c6fc40c1e984f893763437 Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Thu, 2 Dec 2021 01:16:45 +0530 Subject: [PATCH 05/19] Fixed pylint issues --- pylsp/python_lsp.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 06744564..e52576d9 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -95,15 +95,13 @@ def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): if not issubclass(handler_class, PythonLSPServer): raise ValueError('Handler class must be an instance of PythonLSPServer') - #pylint: disable=wrong-import-position + # pylint: disable=import-outside-toplevel # imports needed only for websocket based server from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory from twisted.internet import reactor import ujson as json - #pylint: enable=wrong-import-position - class MyServerProtocol(WebSocketServerProtocol): def handler(self, message): @@ -119,10 +117,13 @@ def handler(self, message): def onOpen(self): log.debug("Creating LSP object") + + # pylint: disable=attribute-defined-outside-init + # Not using default stream reader and writer. # Instead using a consumer based approach to handle processed requests - self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, - check_parent_process=check_parent_process) # pylint: disable=attribute-defined-outside-init + self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, + check_parent_process=check_parent_process) def onMessage(self, payload, isBinary): if isBinary: @@ -139,14 +140,14 @@ def onClose(self, reason): self._lsp.m_shutdown() self._lsp.m_exit() log.debug(f"WS connection closed due to : {reason}!") - + factory = WebSocketServerFactory(f"ws://{bind_addr}:{port}") factory.protocol = MyServerProtocol # factory.setProtocolOptions(maxConnections=2) # This only supports IPv4 connections reactor.listenTCP(port, factory) - + log.info(f"Starting Web Sockets server on port: {port}") # runs forever @@ -169,30 +170,30 @@ def __init__(self, rx, tx, check_parent_process=False, consumer=None): self.uri_workspace_mapper = {} self._check_parent_process = check_parent_process - + if rx is not None: self._jsonrpc_stream_reader = JsonRpcStreamReader(rx) else: self._jsonrpc_stream_reader = None - + if tx is not None: self._jsonrpc_stream_writer = JsonRpcStreamWriter(tx) else: self._jsonrpc_stream_writer = None - + # if Consumer is None, It is assumed that the default streams based approach is being used if consumer is None: self._endpoint = Endpoint(self, self._jsonrpc_stream_writer.write, max_workers=MAX_WORKERS) else: self._endpoint = Endpoint(self, consumer, max_workers=MAX_WORKERS) - + self._dispatchers = [] self._shutdown = False def start(self): """Entry point for the server.""" self._jsonrpc_stream_reader.listen(self._endpoint.consume) - + def consume(self, message): """Entry point for consumer based server. Alternate to stream listeners""" # assuming message will be JSON From 93b731a1384c2289e3f8ce452dae54c7746a7278 Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Thu, 2 Dec 2021 01:19:58 +0530 Subject: [PATCH 06/19] Updated trailing spaces issues --- pylsp/python_lsp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index e52576d9..cbd90c76 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -117,9 +117,9 @@ def handler(self, message): def onOpen(self): log.debug("Creating LSP object") - + # pylint: disable=attribute-defined-outside-init - + # Not using default stream reader and writer. # Instead using a consumer based approach to handle processed requests self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, From 315c8d6b84d9b7efda63697ca9c34bcbfe542424 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Thu, 2 Dec 2021 13:38:15 +0530 Subject: [PATCH 07/19] Fixed pycodestyle checks --- pylsp/__main__.py | 2 +- pylsp/python_lsp.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pylsp/__main__.py b/pylsp/__main__.py index becb0023..e34cbf71 100644 --- a/pylsp/__main__.py +++ b/pylsp/__main__.py @@ -78,7 +78,7 @@ def main(): PythonLSPServer) elif args.ws: start_ws_lang_server(args.host, args.port, args.check_parent_process, - PythonLSPServer) + PythonLSPServer) else: stdin, stdout = _binary_stdio() start_io_lang_server(stdin, stdout, args.check_parent_process, diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index cbd90c76..c3af119c 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -108,7 +108,7 @@ def handler(self, message): """Handler to send responses of processed requests to respective web socket clients""" try: # we have to send bytes to the sendmessage method, So encoding to utf8 - payload = json.dumps(message, ensure_ascii = False).encode('utf8') + payload = json.dumps(message, ensure_ascii=False).encode('utf8') # Binary payload support will be provided in future self.sendMessage(payload, isBinary=False) @@ -123,7 +123,7 @@ def onOpen(self): # Not using default stream reader and writer. # Instead using a consumer based approach to handle processed requests self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, - check_parent_process=check_parent_process) + check_parent_process=check_parent_process) def onMessage(self, payload, isBinary): if isBinary: From e650c90aee30332bb23915b6c98db88b313a88a1 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Fri, 3 Dec 2021 14:28:16 +0530 Subject: [PATCH 08/19] fixed issue with on close misisng parameters --- pylsp/python_lsp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index c3af119c..ea2166c6 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -135,11 +135,11 @@ def onMessage(self, payload, isBinary): log.debug("consuming payload and feeding it LSP handler") self._lsp.consume(json.loads(payload.decode('utf8'))) - def onClose(self, reason): + def onClose(self, wasClean, code, reason): log.debug("shutting down and exiting LSP handler") self._lsp.m_shutdown() self._lsp.m_exit() - log.debug(f"WS connection closed due to : {reason}!") + log.debug(f"WS connection closed: wasClean: {wasClean}, code: {code}, reason: {reason}!") factory = WebSocketServerFactory(f"ws://{bind_addr}:{port}") factory.protocol = MyServerProtocol From 25e3fabab3c5f09f813e00f2a63523a72bbc8206 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Sat, 4 Dec 2021 17:38:22 +0530 Subject: [PATCH 09/19] - moved websockets information to end of configuration - renamed extras - autobahn-twisted to websockets - added more info on the import error --- README.md | 36 +++++++++++++++--------------------- pylsp/python_lsp.py | 9 ++++++--- setup.py | 2 +- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 02105c4d..eee8529c 100644 --- a/README.md +++ b/README.md @@ -21,27 +21,6 @@ If the respective dependencies are found, the following optional providers will - [autopep8](https://github.com/hhatto/autopep8) for code formatting - [YAPF](https://github.com/google/yapf) for code formatting (preferred over autopep8) -PYLSP can now be used in Web Sockets based configuration as follows: - -``` -pylsp --ws --host [host/ip] --port [port] -``` - -The following libraries are required for Web Sockets support: -- [autobahn-twisted](https://github.com/crossbario/autobahn-python) for PYLSP Web sockets using autobahn in twisted configuration. refer [Autobahn installation](https://autobahn.readthedocs.io/en/latest/installation.html) for more details - -you can install these dependencies with below command: - -``` -pip install autobahn[twisted,accelerate]>=21.11.1 -``` - -or with this: - -``` -pip install 'python-lsp-server[autobahn-twisted]' -``` - Optional providers can be installed using the `extras` syntax. To install [YAPF](https://github.com/google/yapf) formatting for example: ``` @@ -93,6 +72,21 @@ To enable pydocstyle for linting docstrings add the following setting in your LS All configuration options are described in [`CONFIGURATION.md`](https://github.com/python-lsp/python-lsp-server/blob/develop/CONFIGURATION.md). +Python LSP Server can communicate over WebSockets when configured as follows: + +``` +pylsp --ws --host [host/ip] --port [port] +``` + +The following libraries are required for Web Sockets support: +- [autobahn-python](https://github.com/crossbario/autobahn-python) for Python LSP Server Web sockets using autobahn with twisted configuration. refer [Autobahn installation](https://autobahn.readthedocs.io/en/latest/installation.html) for more details + +you can install these dependencies with below command: + +``` +pip install 'python-lsp-server[websockets]' +``` + ## LSP Server Features * Auto Completion diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index ea2166c6..015fac1a 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -98,9 +98,12 @@ def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): # pylint: disable=import-outside-toplevel # imports needed only for websocket based server - from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory - from twisted.internet import reactor - import ujson as json + try: + from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory + from twisted.internet import reactor + import ujson as json + except ImportError as e: + raise ImportError("websocket modules missing. Please run pip install 'python-lsp-server[websockets]") from e class MyServerProtocol(WebSocketServerProtocol): diff --git a/setup.py b/setup.py index 142bb165..b7a9c40e 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ def get_version(module='pylsp'): 'pylint': ['pylint>=2.5.0'], 'rope': ['rope>0.10.5'], 'yapf': ['yapf'], - 'autobahn-twisted': ['autobahn[twisted,accelerate]>=21.11.1'], + 'websockets': ['autobahn[twisted,accelerate]>=21.11.1'], 'test': ['pylint>=2.5.0', 'pytest', 'pytest-cov', 'coverage', 'numpy', 'pandas', 'matplotlib', 'pyqt5', 'flaky'], }, From 8b1cb599b7ddef58015f652b7e28e534ee622731 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda Date: Sun, 5 Dec 2021 07:29:10 +0530 Subject: [PATCH 10/19] removed unnecessary comments, updated comment --- pylsp/python_lsp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 015fac1a..60c74023 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -146,7 +146,6 @@ def onClose(self, wasClean, code, reason): factory = WebSocketServerFactory(f"ws://{bind_addr}:{port}") factory.protocol = MyServerProtocol - # factory.setProtocolOptions(maxConnections=2) # This only supports IPv4 connections reactor.listenTCP(port, factory) @@ -184,7 +183,7 @@ def __init__(self, rx, tx, check_parent_process=False, consumer=None): else: self._jsonrpc_stream_writer = None - # if Consumer is None, It is assumed that the default streams based approach is being used + # if Consumer is None, it is assumed that the default streams-based approach is being used if consumer is None: self._endpoint = Endpoint(self, self._jsonrpc_stream_writer.write, max_workers=MAX_WORKERS) else: From a43e4a17ed0d3fed9d2f3fd33d7697423c4ce2e4 Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Tue, 15 Feb 2022 12:14:24 +0530 Subject: [PATCH 11/19] Updated review comments --- pylsp/python_lsp.py | 4 ++-- setup.cfg | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 60c74023..08d6b583 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -6,6 +6,7 @@ import os import socketserver import threading +import ujson as json from pylsp_jsonrpc.dispatchers import MethodDispatcher from pylsp_jsonrpc.endpoint import Endpoint @@ -101,7 +102,6 @@ def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): try: from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory from twisted.internet import reactor - import ujson as json except ImportError as e: raise ImportError("websocket modules missing. Please run pip install 'python-lsp-server[websockets]") from e @@ -183,7 +183,7 @@ def __init__(self, rx, tx, check_parent_process=False, consumer=None): else: self._jsonrpc_stream_writer = None - # if Consumer is None, it is assumed that the default streams-based approach is being used + # if consumer is None, it is assumed that the default streams-based approach is being used if consumer is None: self._endpoint = Endpoint(self, self._jsonrpc_stream_writer.write, max_workers=MAX_WORKERS) else: diff --git a/setup.cfg b/setup.cfg index b9782ddb..e396fe11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,6 @@ all = pylint>=2.5.0 rope>=0.10.5 yapf - autobahn[twisted,accelerate]>=21.11.1 autopep8 = autopep8>=1.6.0,<1.7.0 flake8 = flake8>=4.0.0,<4.1.0 mccabe = mccabe>=0.6.0,<0.7.0 @@ -43,7 +42,7 @@ pyflakes = pyflakes>=2.4.0,<2.5.0 pylint = pylint>=2.5.0 rope = rope>0.10.5 yapf = yapf -websockets = autobahn[twisted,accelerate]>=21.11.1 +websockets = autobahn[twisted,accelerate]>=22.1.1 test = pylint>=2.5.0 pytest From a977e3368091e11ceff24498ff520cf2347208ba Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:19:35 +0530 Subject: [PATCH 12/19] Websockets (#4) * Added websockets based implementation to reduce libraries and 3rd party dependencies * Removed message encode and decode as they are no longer needed Co-authored-by: Pradeep Neerukonda --- README.md | 2 +- pylsp/__main__.py | 2 +- pylsp/python_lsp.py | 83 ++++++++++++++++++++------------------------- setup.cfg | 2 +- 4 files changed, 39 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 2f21daaf..ea62bd19 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ pylsp --ws --host [host/ip] --port [port] ``` The following libraries are required for Web Sockets support: -- [autobahn-python](https://github.com/crossbario/autobahn-python) for Python LSP Server Web sockets using autobahn with twisted configuration. refer [Autobahn installation](https://autobahn.readthedocs.io/en/latest/installation.html) for more details +- [websockets](https://websockets.readthedocs.io/en/stable/) for Python LSP Server Web sockets using websockets library. refer [Websockets installation](https://websockets.readthedocs.io/en/stable/intro/index.html#installation) for more details you can install these dependencies with below command: diff --git a/pylsp/__main__.py b/pylsp/__main__.py index e34cbf71..50950a30 100644 --- a/pylsp/__main__.py +++ b/pylsp/__main__.py @@ -77,7 +77,7 @@ def main(): start_tcp_lang_server(args.host, args.port, args.check_parent_process, PythonLSPServer) elif args.ws: - start_ws_lang_server(args.host, args.port, args.check_parent_process, + start_ws_lang_server(args.port, args.check_parent_process, PythonLSPServer) else: stdin, stdout = _binary_stdio() diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 9e0c4704..3f09e90f 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -92,68 +92,57 @@ def start_io_lang_server(rfile, wfile, check_parent_process, handler_class): server.start() -def start_ws_lang_server(bind_addr, port, check_parent_process, handler_class): +def start_ws_lang_server(port, check_parent_process, handler_class): if not issubclass(handler_class, PythonLSPServer): raise ValueError('Handler class must be an instance of PythonLSPServer') # pylint: disable=import-outside-toplevel - # imports needed only for websocket based server + # imports needed only for websockets based server try: - from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory - from twisted.internet import reactor + import asyncio + from concurrent.futures import ThreadPoolExecutor + import websockets except ImportError as e: raise ImportError("websocket modules missing. Please run pip install 'python-lsp-server[websockets]") from e - class MyServerProtocol(WebSocketServerProtocol): - def handler(self, message): - """Handler to send responses of processed requests to respective web socket clients""" - try: - # we have to send bytes to the sendmessage method, So encoding to utf8 - payload = json.dumps(message, ensure_ascii=False).encode('utf8') - - # Binary payload support will be provided in future - self.sendMessage(payload, isBinary=False) - except Exception as e: # pylint: disable=broad-except - log.exception("Failed to write message to output file %s, %s", payload, str(e)) - - def onOpen(self): - log.debug("Creating LSP object") + async def pylsp_ws(websocket): + log.debug("Creating LSP object") - # pylint: disable=attribute-defined-outside-init + # creating a partial function and suppling the websocket connection + response_handler = partial(send_message, websocket = websocket) - # Not using default stream reader and writer. - # Instead using a consumer based approach to handle processed requests - self._lsp = handler_class(rx=None, tx=None, consumer=self.handler, + # Not using default stream reader and writer. + # Instead using a consumer based approach to handle processed requests + pylsp_handler = handler_class(rx=None, tx=None, consumer=response_handler, check_parent_process=check_parent_process) - def onMessage(self, payload, isBinary): - if isBinary: - # binary message support will be provided in future - # print(f"Binary message received: {len(payload)} bytes") - pass - else: - # we get payload as byte array and so decoding it to string using utf8 - log.debug("consuming payload and feeding it LSP handler") - self._lsp.consume(json.loads(payload.decode('utf8'))) - - def onClose(self, wasClean, code, reason): - log.debug("shutting down and exiting LSP handler") - self._lsp.m_shutdown() - self._lsp.m_exit() - log.debug(f"WS connection closed: wasClean: {wasClean}, code: {code}, reason: {reason}!") - - factory = WebSocketServerFactory(f"ws://{bind_addr}:{port}") - factory.protocol = MyServerProtocol + tpool = ThreadPoolExecutor(max_workers=5) + loop = asyncio.get_running_loop() - # This only supports IPv4 connections - reactor.listenTCP(port, factory) - - log.info(f"Starting Web Sockets server on port: {port}") - - # runs forever - reactor.run() + async for message in websocket: + try: + log.debug("consuming payload and feeding it LSP handler") + request = json.loads(message) + loop.run_in_executor(tpool, pylsp_handler.consume, request) + except Exception as e: # pylint: disable=broad-except + log.exception("Failed to process request %s, %s", message, str(e)) + + def send_message(message, websocket): + """Handler to send responses of processed requests to respective web socket clients""" + try: + payload = json.dumps(message, ensure_ascii=False) + asyncio.run(websocket.send(payload)) + except Exception as e: # pylint: disable=broad-except + log.exception("Failed to write message %s, %s", message, str(e)) + + async def run_server(): + async with websockets.serve(pylsp_ws, port=port): + # runs forever + await asyncio.Future() + + asyncio.run(run_server()) class PythonLSPServer(MethodDispatcher): diff --git a/setup.cfg b/setup.cfg index e396fe11..6d47be8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ pyflakes = pyflakes>=2.4.0,<2.5.0 pylint = pylint>=2.5.0 rope = rope>0.10.5 yapf = yapf -websockets = autobahn[twisted,accelerate]>=22.1.1 +websockets = websockets>=10.2 test = pylint>=2.5.0 pytest From 1de7296e09f4522f884b708388b153d2f37aa160 Mon Sep 17 00:00:00 2001 From: pradeep <16036792+npradeep357@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:21:17 +0530 Subject: [PATCH 13/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea62bd19..45303532 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ As an example, to change the list of errors that pycodestyle will ignore, assumi Python LSP Server can communicate over WebSockets when configured as follows: ``` -pylsp --ws --host [host/ip] --port [port] +pylsp --ws --port [port] ``` The following libraries are required for Web Sockets support: From 330bab60dc78bb643f9413837358b2aacc1f9f4e Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:28:07 +0530 Subject: [PATCH 14/19] removed triling whitespaces --- pylsp/python_lsp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 3f09e90f..0ce3991d 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -128,7 +128,7 @@ async def pylsp_ws(websocket): loop.run_in_executor(tpool, pylsp_handler.consume, request) except Exception as e: # pylint: disable=broad-except log.exception("Failed to process request %s, %s", message, str(e)) - + def send_message(message, websocket): """Handler to send responses of processed requests to respective web socket clients""" try: @@ -141,7 +141,7 @@ async def run_server(): async with websockets.serve(pylsp_ws, port=port): # runs forever await asyncio.Future() - + asyncio.run(run_server()) From 0cb47b5d5540fa866b1508b3103a0de45736b3d2 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:39:22 +0530 Subject: [PATCH 15/19] removed space between parameters --- pylsp/python_lsp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 0ce3991d..edc9a116 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -106,12 +106,11 @@ def start_ws_lang_server(port, check_parent_process, handler_class): except ImportError as e: raise ImportError("websocket modules missing. Please run pip install 'python-lsp-server[websockets]") from e - async def pylsp_ws(websocket): log.debug("Creating LSP object") # creating a partial function and suppling the websocket connection - response_handler = partial(send_message, websocket = websocket) + response_handler = partial(send_message, websocket=websocket) # Not using default stream reader and writer. # Instead using a consumer based approach to handle processed requests From 297ab9e2333c07e03d80be2d3583285d930dbe70 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:48:07 +0530 Subject: [PATCH 16/19] Updated debug log --- pylsp/python_lsp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index edc9a116..144a0781 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -122,7 +122,7 @@ async def pylsp_ws(websocket): async for message in websocket: try: - log.debug("consuming payload and feeding it LSP handler") + log.debug("consuming payload and feeding it to LSP handler") request = json.loads(message) loop.run_in_executor(tpool, pylsp_handler.consume, request) except Exception as e: # pylint: disable=broad-except From 294e26732022b41b5b53345d423e7d02d20a2b8b Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Fri, 22 Apr 2022 11:18:56 +0530 Subject: [PATCH 17/19] changed logic to use single threadpool excutor --- pylsp/python_lsp.py | 57 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 144a0781..7ecb5924 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -106,42 +106,41 @@ def start_ws_lang_server(port, check_parent_process, handler_class): except ImportError as e: raise ImportError("websocket modules missing. Please run pip install 'python-lsp-server[websockets]") from e - async def pylsp_ws(websocket): - log.debug("Creating LSP object") + with ThreadPoolExecutor(max_workers=10) as tpool: + async def pylsp_ws(websocket): + log.debug("Creating LSP object") - # creating a partial function and suppling the websocket connection - response_handler = partial(send_message, websocket=websocket) + # creating a partial function and suppling the websocket connection + response_handler = partial(send_message, websocket=websocket) - # Not using default stream reader and writer. - # Instead using a consumer based approach to handle processed requests - pylsp_handler = handler_class(rx=None, tx=None, consumer=response_handler, - check_parent_process=check_parent_process) + # Not using default stream reader and writer. + # Instead using a consumer based approach to handle processed requests + pylsp_handler = handler_class(rx=None, tx=None, consumer=response_handler, + check_parent_process=check_parent_process) - tpool = ThreadPoolExecutor(max_workers=5) - loop = asyncio.get_running_loop() - - async for message in websocket: + async for message in websocket: + try: + log.debug("consuming payload and feeding it to LSP handler") + request = json.loads(message) + loop = asyncio.get_running_loop() + await loop.run_in_executor(tpool, pylsp_handler.consume, request) + except Exception as e: # pylint: disable=broad-except + log.exception("Failed to process request %s, %s", message, str(e)) + + def send_message(message, websocket): + """Handler to send responses of processed requests to respective web socket clients""" try: - log.debug("consuming payload and feeding it to LSP handler") - request = json.loads(message) - loop.run_in_executor(tpool, pylsp_handler.consume, request) + payload = json.dumps(message, ensure_ascii=False) + asyncio.run(websocket.send(payload)) except Exception as e: # pylint: disable=broad-except - log.exception("Failed to process request %s, %s", message, str(e)) - - def send_message(message, websocket): - """Handler to send responses of processed requests to respective web socket clients""" - try: - payload = json.dumps(message, ensure_ascii=False) - asyncio.run(websocket.send(payload)) - except Exception as e: # pylint: disable=broad-except - log.exception("Failed to write message %s, %s", message, str(e)) + log.exception("Failed to write message %s, %s", message, str(e)) - async def run_server(): - async with websockets.serve(pylsp_ws, port=port): - # runs forever - await asyncio.Future() + async def run_server(): + async with websockets.serve(pylsp_ws, port=port): + # runs forever + await asyncio.Future() - asyncio.run(run_server()) + asyncio.run(run_server()) class PythonLSPServer(MethodDispatcher): From 36c07be70b9367d1ffefffd1c82bd649d7914190 Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Tue, 17 May 2022 10:52:51 +0530 Subject: [PATCH 18/19] updated websockets version to 10.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 601e59e0..708df34b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ pyflakes = ["pyflakes>=2.4.0,<2.5.0"] pylint = ["pylint>=2.5.0"] rope = ["rope>0.10.5"] yapf = ["yapf"] -websockets = websockets>=10.2 +websockets = ["websockets>=10.3"] test = [ "pylint>=2.5.0", "pytest", From 4f646657e32e59b5184356be9730dc1a93a6ff2f Mon Sep 17 00:00:00 2001 From: Pradeep Neerukonda <16036792+npradeep357@users.noreply.github.com> Date: Mon, 30 May 2022 13:04:18 +0530 Subject: [PATCH 19/19] Addressing comments --- README.md | 2 +- pylsp/python_lsp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2ca1d7c7..d2dcd5fc 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ pylsp --ws --port [port] The following libraries are required for Web Sockets support: - [websockets](https://websockets.readthedocs.io/en/stable/) for Python LSP Server Web sockets using websockets library. refer [Websockets installation](https://websockets.readthedocs.io/en/stable/intro/index.html#installation) for more details -you can install these dependencies with below command: +You can install this dependency with command below: ``` pip install 'python-lsp-server[websockets]' diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 7ecb5924..8cac63d5 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -184,7 +184,7 @@ def start(self): self._jsonrpc_stream_reader.listen(self._endpoint.consume) def consume(self, message): - """Entry point for consumer based server. Alternate to stream listeners""" + """Entry point for consumer based server. Alternative to stream listeners.""" # assuming message will be JSON self._endpoint.consume(message)