Adding json rpc library with tests

doc/README.md

@@ -0,0 +1,30 @@
+1.	Install the dependencies via pip with the script below.
+  ```Shell
+  python scripts/dev_setup.py
+
+2.  Add `<clone root>\src` to your PYTHONPATH environment variable:
+
+  #####Windows
+  ```BatchFile
+  set PYTHONPATH=<clone root>\src;%PYTHONPATH%
+  ```
+  #####OSX/Ubuntu (bash)
+  ```Shell
+  export PYTHONPATH=<clone root>/src:${PYTHONPATH}
+
+##Running Tests:
+####Command line
+#####Windows:
+  Provided your PYTHONPATH was set correctly, you can run the tests from your `<root clone>` directory.
+
+  To test the common modules of the CLI:
+  ```BatchFile
+  python -m unittest discover -s src/common/tests
+  ```
+
+  To test the scripter module of the CLI:
+  ```BatchFile
+  python -m unittest discover -s src/mssqlscripter/mssql/scripter/tests
+  ```
+
+  Additionally, you can run tests for all CLI tools and common modules using the `Run_All_Tests.bat` script.

requirements.txt

@@ -0,0 +1,3 @@
+enum34==1.1.6
+pip==9.0.1
+setuptools==30.4.0

scripts/README.md

@@ -0,0 +1,3 @@
+# Contributing
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [[email protected]](mailto:[email protected]) with any additional questions or comments.

scripts/Run_All_Tests

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+python -m unittest discover -s ../src/common/tests
+python -m unittest discover -s ../src/mssql-scripter/mssql/scripter/tests

scripts/Run_All_Tests.bat

@@ -0,0 +1,9 @@
+@echo off
+
+REM --------------------------------------------------------------------------------------------
+REM Copyright (c) Microsoft Corporation. All rights reserved.
+REM Licensed under the MIT License. See License.txt in the project root for license information.
+REM --------------------------------------------------------------------------------------------
+
+python -m unittest discover -s ../src/common/tests
+python -m unittest discover -s ../src/mssql-scripter/mssql/scripter/tests

scripts/dev_setup.py

@@ -0,0 +1,29 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import sys
+import os
+from subprocess import check_call, CalledProcessError
+
+root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..'))
+
+def exec_command(command):
+    try:
+        print('Executing: ' + command)
+        check_call(command.split(), cwd=root_dir)
+        print()
+    except CalledProcessError as err:
+        print(err, file=sys.stderr)
+        sys.exit(1)
+
+print('Running dev setup...')
+print('Root directory \'{}\'\n'.format(root_dir))
+
+# install general requirements
+exec_command('pip install -r requirements.txt')
+
+print('Finished dev setup.')

src/common/README.md

@@ -0,0 +1,2 @@
+Microsoft XPlat CLI common module
+=================================

src/common/__init__.py

@@ -0,0 +1,4 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------

src/common/json_rpc.py

@@ -0,0 +1,223 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from io import BytesIO
+from enum import Enum
+import json
+
+class Read_State(Enum):
+    Header = 1
+    Content = 2
+
+class Json_Rpc_Writer(object):
+    """
+    Writes to the supplied stream through the JSON RPC Protocol where a request is formatted through a method
+    name and the necessary parameters.
+    """
+    HEADER = 'Content-Length: {0}\r\n\r\n'
+
+    def __init__(self, stream, encoding = None):
+        self.stream = stream
+        self.encoding = encoding
+        if encoding is None:
+            self.encoding = 'UTF-8'
+
+    def send_request(self, method, params, id = None):
+        """
+        Forms and writes a JSON RPC protocol compliant request a method and it's parameters to the stream.
+        """
+        # Perhaps move to a different def to add some validation
+        content_body = {
+            'jsonrpc': '2.0',
+            'method': method,
+            'params': params,
+            'id': id 
+        }
+
+        json_content = json.dumps(content_body)
+        header = self.HEADER.format(str(len(json_content)))
+
+        self.stream.write(header.encode('ascii'))
+        self.stream.write(json_content.encode(self.encoding))
+        self.stream.flush()
+
+class Json_Rpc_Reader(object):
+    """
+    Reads from the supplied stream through the JSON RPC Protocol. A Content-length header is required in the format
+    of "Content-Length: <number of bytes>".
+    Various exceptions may occur during the read process and are documented in each method.
+    """
+    # \r\n
+    CR = 13
+    LF = 10
+    BUFFER_RESIZE_TRIGGER= 0.25
+    DEFAULT_BUFFER_SIZE = 8192
+
+    def __init__(self, stream, encoding = None):
+        self.encoding = encoding
+        if encoding is None:
+            self.encoding = 'UTF-8'
+
+        self.stream = stream
+        self.buffer = bytearray(self.DEFAULT_BUFFER_SIZE)       
+        # Pointer to end of buffer content
+        self.buffer_end_offset = 0
+        # Pointer to where we have read up to
+        self.read_offset = 0
+        self.expected_content_length = 0
+        self.headers = {}
+        self.read_state = Read_State.Header
+
+    def read_response(self):
+        """
+        Reads the response from the supplied stream by chunks into a buffer until all headers and body content are read. 
+
+        Returns the response body content in JSON
+        Exceptions raised:
+            ValueError
+                if the body-content can not be serialized to a JSON object
+        """
+        # Using a mutable list to hold the value since a immutable string passed by reference won't change the value
+        content = ['']
+        while (self.read_next_chunk()):
+            # If we can't read a header, read the next chunk
+            if (self.read_state is Read_State.Header and not self.try_read_headers()):
+                continue
+            # If we read the header, try the content. If that fails, read the next chunk
+            if (self.read_state is Read_State.Content and not self.try_read_content(content)):
+                continue
+            # We have the  content
+            break
+
+        # Resize buffer and remove bytes we have read
+        self.trim_buffer_and_resize(self.read_offset)
+        try:
+            return json.loads(content[0])
+        except ValueError:
+            # response has invalid json object, throw Exception TODO: log message to telemetry
+            raise
+
+    def read_next_chunk(self):
+        """
+        Reads a chunk of the stream into the byte array. Buffer size is doubled if less than 25% of buffer space is available.abs
+        Exceptions raised:
+            EOFError
+                Stream was empty or Stream did not contain a valid header or content-body
+            IOError
+                Stream was closed externally
+        """
+        # Check if we need to resize
+        current_buffer_size = len(self.buffer)
+        if ((current_buffer_size - float(self.buffer_end_offset)) / current_buffer_size) < self.BUFFER_RESIZE_TRIGGER:
+            resized_buffer = bytearray(current_buffer_size * 2)
+            # copy current buffer content to new buffer
+            resized_buffer[0:current_buffer_size] = self.buffer
+            # point to new buffer
+            self.buffer = resized_buffer
+
+        # read next chunk into buffer
+        # Memory view is required in order to read into a subset of a byte array
+        try:
+            length_read = self.stream.readinto(memoryview(self.buffer)[self.buffer_end_offset:])
+            self.buffer_end_offset += length_read
+
+            if (length_read == 0):
+                # Nothing was read, could be due to the server process shutting down while leaving stream open
+                raise EOFError("End of stream reached with no valid header or content-body")
+
+            return True
+
+        except IOError as error:
+            # TODO: add to telemetry
+            raise
+
+    def try_read_headers(self):
+        """
+        Attempts to read the Header information from the internal buffer expending the last header contain "\r\n\r\n.
+
+        Returns false if the header was not found.
+        Exceptions:
+            LookupError The content-length header was not found
+            ValueError The content-length contained a invalid literal for int
+        """
+        # Scan the buffer up until right before the CRLFCRLF
+        scan_offset = self.read_offset
+        while(scan_offset + 3 < self.buffer_end_offset and
+                (self.buffer[scan_offset] != self.CR or
+                self.buffer[scan_offset + 1] != self.LF or
+                self.buffer[scan_offset + 2] != self.CR or
+                self.buffer[scan_offset + 3] != self.LF)):
+            scan_offset += 1
+
+        # if we reached the end
+        if (scan_offset + 3 >= self.buffer_end_offset ):
+            return False
+
+        # Split the headers by new line
+        try:
+            headers_read = self.buffer[self.read_offset:scan_offset].decode('ascii').split('\n')
+            for header in headers_read:
+                colon_index = header.find(':')
+
+                if (colon_index == -1):
+                    raise KeyError('Colon missing from Header: {0}.'.format(header))
+
+                # Making all headers lowercase to support case insensitivity
+                header_key = header[:colon_index].lower()
+                header_value = header[colon_index + 1:]
+
+                self.headers[header_key] = header_value
+
+            #Find content body in the list of headers and parse the Value
+            if (not ('content-length' in self.headers)):
+                raise LookupError('Content-Length was not found in headers received.')
+
+            self.expected_content_length = int(self.headers['content-length'])
+
+        except ValueError:
+            # Content-length contained invalid literal for int
+            self.trim_buffer_and_resize(scan_offset + 4)
+            raise
+
+        # Pushing read pointer past the newline characters
+        self.read_offset = scan_offset + 4
+        self.read_state = Read_State.Content
+
+        return True
+
+    def try_read_content(self, content):
+        """
+        Attempts to read the content from the internal buffer.
+
+        Returns false if buffer does not contain the entire content.
+        """
+        # if we buffered less than the expected content length, return false
+        if (self.buffer_end_offset - self.read_offset < self.expected_content_length):
+            return False
+
+        content[0] = self.buffer[self.read_offset:self.read_offset + self.expected_content_length].decode(self.encoding)
+        self.read_offset += self.expected_content_length
+
+        self.read_state = Read_State.Header
+        return True
+
+    def trim_buffer_and_resize(self, bytes_to_remove):
+        """
+        Trims the buffer by the passed in bytes_to_remove by creating a new buffer that is at a minimum the default max size.
+        """
+        current_buffer_size = len(self.buffer)
+        # Create a new buffer with either minumum size or leftover size
+        new_buffer = bytearray(max(current_buffer_size - bytes_to_remove, self.DEFAULT_BUFFER_SIZE))
+
+        # if we have content we did not read, copy that portion to the new buffer
+        if (bytes_to_remove <= current_buffer_size):
+            new_buffer[:self.buffer_end_offset - bytes_to_remove] = self.buffer[bytes_to_remove:self.buffer_end_offset]
+
+        # Point to the new buffer
+        self.buffer = new_buffer
+
+        # reset pointers after the shift
+        self.read_offset = 0
+        self.buffer_end_offset -= bytes_to_remove