From dbad6742517e616fe5e0046895f6664345e0aa25 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 31 Mar 2020 23:54:20 -0700 Subject: [PATCH 01/37] Initial commit for POC. This is WIP --- .../botbuilder/core/streaming/__init__.py | 7 ++ .../bot_framework_http_adapter_base.py | 18 ++++ .../streaming/streaming_activity_processor.py | 21 +++++ .../streaming/streaming_request_handler.py | 8 ++ libraries/botbuilder-core/setup.py | 1 + .../integration/aiohttp/__init__.py | 2 + .../aiohttp/bot_framework_http_adapter.py | 26 ++++++ libraries/botbuilder-streaming/README.rst | 83 ++++++++++++++++++ .../botbuilder/streaming/__init__.py | 12 +++ .../botbuilder/streaming/about.py | 14 +++ .../botbuilder/streaming/payloads/__init__.py | 6 ++ .../payloads/response_message_stream.py | 10 +++ .../botbuilder/streaming/receive_request.py | 11 +++ .../botbuilder/streaming/request_handler.py | 15 ++++ .../streaming/streaming_response.py | 33 +++++++ .../botbuilder-streaming/requirements.txt | 4 + libraries/botbuilder-streaming/setup.py | 44 ++++++++++ .../experimental/streamming-extensions/app.py | 87 +++++++++++++++++++ .../streamming-extensions/bots/__init__.py | 6 ++ .../streamming-extensions/bots/echo_bot.py | 19 ++++ .../streamming-extensions/config.py | 15 ++++ 21 files changed, 442 insertions(+) create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/__init__.py create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/streaming_activity_processor.py create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py create mode 100644 libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py create mode 100644 libraries/botbuilder-streaming/README.rst create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/about.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py create mode 100644 libraries/botbuilder-streaming/requirements.txt create mode 100644 libraries/botbuilder-streaming/setup.py create mode 100644 samples/experimental/streamming-extensions/app.py create mode 100644 samples/experimental/streamming-extensions/bots/__init__.py create mode 100644 samples/experimental/streamming-extensions/bots/echo_bot.py create mode 100644 samples/experimental/streamming-extensions/config.py diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py new file mode 100644 index 000000000..efc2a34bd --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .bot_framework_http_adapter_base import BotFrameworkHttpAdapterBase +from .streaming_activity_processor import StreamingActivityProcessor + +__all__ = ["BotFrameworkHttpAdapterBase", "StreamingActivityProcessor"] diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py new file mode 100644 index 000000000..09ee9d9d6 --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from botbuilder.core import Bot, BotFrameworkAdapter, BotFrameworkAdapterSettings +from botframework.connector.auth import ClaimsIdentity + +from .streaming_activity_processor import StreamingActivityProcessor + + +class BotFrameworkHttpAdapterBase(BotFrameworkAdapter, StreamingActivityProcessor): + def __init__(self, settings: BotFrameworkAdapterSettings): + super().__init__(self, settings) + + self._connected_bot: Bot = None + self._claims_identity: ClaimsIdentity = None + self._request_handlers: List[object] = None diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_activity_processor.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_activity_processor.py new file mode 100644 index 000000000..6b6f16893 --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_activity_processor.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC +from typing import Awaitable, Callable + +from botbuilder.core import TurnContext, InvokeResponse +from botbuilder.schema import Activity + + +class StreamingActivityProcessor(ABC): + """ + Process streaming activities. + """ + + async def process_streaming_activity( + self, + activity: Activity, + bot_callback_handler: Callable[[TurnContext], Awaitable], + ) -> InvokeResponse: + raise NotImplementedError() diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py new file mode 100644 index 000000000..5efb2ffa5 --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from botbuilder.core import bot + + +class StreamingRequestHandler: + pass diff --git a/libraries/botbuilder-core/setup.py b/libraries/botbuilder-core/setup.py index 21a356b49..60b884104 100644 --- a/libraries/botbuilder-core/setup.py +++ b/libraries/botbuilder-core/setup.py @@ -37,6 +37,7 @@ "botbuilder.core.inspection", "botbuilder.core.integration", "botbuilder.core.skills", + "botbuilder.core.streaming", "botbuilder.core.teams", ], install_requires=REQUIRES, diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/__init__.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/__init__.py index 1bb31e665..d43aa50fa 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/__init__.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/__init__.py @@ -8,9 +8,11 @@ from .aiohttp_channel_service import aiohttp_channel_service_routes from .aiohttp_channel_service_exception_middleware import aiohttp_error_middleware from .bot_framework_http_client import BotFrameworkHttpClient +from .bot_framework_http_adapter import BotFrameworkHttpAdapter __all__ = [ "aiohttp_channel_service_routes", "aiohttp_error_middleware", "BotFrameworkHttpClient", + "BotFrameworkHttpAdapter", ] diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py new file mode 100644 index 000000000..d601123ff --- /dev/null +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -0,0 +1,26 @@ +from aiohttp.web import Request, Response, WebSocketResponse +from botbuilder.core import BotFrameworkAdapter, Bot + + +class BotFrameworkHttpAdapter(BotFrameworkAdapter): + async def process(self, request: Request, ws_response: WebSocketResponse, bot: Bot): + if not request: + raise TypeError("request can't be None") + if not ws_response: + raise TypeError("ws_response can't be None") + if not bot: + raise TypeError("bot can't be None") + + if request.method == "GET": + await self.connect_web_socket(bot, request, ws_response) + + async def connect_web_socket( + self, bot: Bot, request: Request, ws_response: WebSocketResponse + ): + if not request: + raise TypeError("request can't be None") + if not ws_response: + raise TypeError("ws_response can't be None") + + if not ws_response.can_prepare(request): + raise Exception("WS not available") diff --git a/libraries/botbuilder-streaming/README.rst b/libraries/botbuilder-streaming/README.rst new file mode 100644 index 000000000..a2bfe44c7 --- /dev/null +++ b/libraries/botbuilder-streaming/README.rst @@ -0,0 +1,83 @@ + +=============================== +BotBuilder-Streaming for Python +=============================== + +.. image:: https://fuselabs.visualstudio.com/SDK_v4/_apis/build/status/Python/SDK_v4-Python-CI?branchName=master + :target: https://fuselabs.visualstudio.com/SDK_v4/_apis/build/status/Python/SDK_v4-Python-CI + :align: right + :alt: Azure DevOps status for master branch +.. image:: https://badge.fury.io/py/botbuilder-core.svg + :target: https://badge.fury.io/py/botbuilder-core + :alt: Latest PyPI package version + +Streaming Extensions libraries for BotBuilder. + +How to Install +============== + +.. code-block:: python + + pip install botbuilder-streaming + + +Documentation/Wiki +================== + +You can find more information on the botbuilder-python project by visiting our `Wiki`_. + +Requirements +============ + +* `Python >= 3.7.0`_ + + +Source Code +=========== +The latest developer version is available in a github repository: +https://github.com/Microsoft/botbuilder-python/ + + +Contributing +============ + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the `Microsoft Open Source Code of Conduct`_. +For more information see the `Code of Conduct FAQ`_ or +contact `opencode@microsoft.com`_ with any additional questions or comments. + +Reporting Security Issues +========================= + +Security issues and bugs should be reported privately, via email, to the Microsoft Security +Response Center (MSRC) at `secure@microsoft.com`_. You should +receive a response within 24 hours. If for some reason you do not, please follow up via +email to ensure we received your original message. Further information, including the +`MSRC PGP`_ key, can be found in +the `Security TechCenter`_. + +License +======= + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT_ License. + +.. _Wiki: https://github.com/Microsoft/botbuilder-python/wiki +.. _Python >= 3.7.0: https://www.python.org/downloads/ +.. _MIT: https://github.com/Microsoft/vscode/blob/master/LICENSE.txt +.. _Microsoft Open Source Code of Conduct: https://opensource.microsoft.com/codeofconduct/ +.. _Code of Conduct FAQ: https://opensource.microsoft.com/codeofconduct/faq/ +.. _opencode@microsoft.com: mailto:opencode@microsoft.com +.. _secure@microsoft.com: mailto:secure@microsoft.com +.. _MSRC PGP: https://technet.microsoft.com/en-us/security/dn606155 +.. _Security TechCenter: https://github.com/Microsoft/vscode/blob/master/LICENSE.txt + +.. `_ \ No newline at end of file diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py new file mode 100644 index 000000000..6acbe2231 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -0,0 +1,12 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from .receive_request import ReceiveRequest +from .request_handler import RequestHandler +from .streaming_response import StreamingResponse + +__all__ = ["ReceiveRequest", "RequestHandler", "StreamingResponse"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/about.py b/libraries/botbuilder-streaming/botbuilder/streaming/about.py new file mode 100644 index 000000000..3cd35b078 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/about.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os + +__title__ = "botbuilder-streaming" +__version__ = ( + os.environ["packageVersion"] if "packageVersion" in os.environ else "4.7.1" +) +__uri__ = "https://www.github.com/Microsoft/botbuilder-python" +__author__ = "Microsoft" +__description__ = "Microsoft Bot Framework Bot Builder" +__summary__ = "Microsoft Bot Framework Bot Builder SDK for Python." +__license__ = "MIT" diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py new file mode 100644 index 000000000..e964074d9 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .response_message_stream import ResponseMessageStream + +__all__ = ["ResponseMessageStream"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py new file mode 100644 index 000000000..716f604b7 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import uuid4, UUID + + +class ResponseMessageStream: + def __init__(self, *, id: UUID = None, content: object = None): + self.id = id or uuid4() + self.content = content diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py new file mode 100644 index 000000000..da0738698 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + + +class ReceiveRequest: + def __init__(self, *, verb: str = None, path: str = None, streams: List[object]): + self.verb = verb + self.path = path + self.streams = streams or [] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py b/libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py new file mode 100644 index 000000000..3214eb7a1 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC +from logging import Logger + +from .receive_request import ReceiveRequest +from .streaming_response import StreamingResponse + + +class RequestHandler(ABC): + async def process_request( + self, request: ReceiveRequest, logger: Logger, context: object + ) -> StreamingResponse: + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py new file mode 100644 index 000000000..e23187860 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +# TODO: reconsider to absolute import +from .payloads import ResponseMessageStream + + +class StreamingResponse: + def __init__( + self, *, status_code: int = None, streams: List[ResponseMessageStream] = None + ): + self.status_code = status_code + self.streams = streams + + def add_stream(self, content: object): + if not content: + raise TypeError("content can't be None") + + if self.streams is None: + self.streams: List[ResponseMessageStream] = [] + + self.streams.append(ResponseMessageStream(content=content)) + + @staticmethod + def create_response(status_code: int, body: object) -> "StreamingResponse": + response = StreamingResponse(status_code=status_code) + + if body: + response.add_stream(body) + + return response diff --git a/libraries/botbuilder-streaming/requirements.txt b/libraries/botbuilder-streaming/requirements.txt new file mode 100644 index 000000000..1d6f7ab31 --- /dev/null +++ b/libraries/botbuilder-streaming/requirements.txt @@ -0,0 +1,4 @@ +msrest==0.6.10 +botframework-connector>=4.7.1 +botbuilder-schema>=4.7.1 +aiohttp>=3.6.2 \ No newline at end of file diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botbuilder-streaming/setup.py new file mode 100644 index 000000000..0a5d36fbd --- /dev/null +++ b/libraries/botbuilder-streaming/setup.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +from setuptools import setup + +VERSION = os.environ["packageVersion"] if "packageVersion" in os.environ else "4.7.1" +REQUIRES = [ + "botbuilder-schema>=4.7.1", + "botframework-connector>=4.7.1", + "botbuilder-core>=4.7.1", +] + +root = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(root, "botbuilder", "streaming", "about.py")) as f: + package_info = {} + info = f.read() + exec(info, package_info) + +with open(os.path.join(root, "README.rst"), encoding="utf-8") as f: + long_description = f.read() + +setup( + name=package_info["__title__"], + version=package_info["__version__"], + url=package_info["__uri__"], + author=package_info["__author__"], + description=package_info["__description__"], + keywords=["BotBuilderStreaming", "bots", "ai", "botframework", "botbuilder",], + long_description=long_description, + long_description_content_type="text/x-rst", + license=package_info["__license__"], + packages=["botbuilder.streaming", "botbuilder.streaming.payloads"], + install_requires=REQUIRES, + classifiers=[ + "Programming Language :: Python :: 3.7", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], +) diff --git a/samples/experimental/streamming-extensions/app.py b/samples/experimental/streamming-extensions/app.py new file mode 100644 index 000000000..450c22b17 --- /dev/null +++ b/samples/experimental/streamming-extensions/app.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import sys +import traceback +from datetime import datetime + +from aiohttp import web +from aiohttp.web import Request, Response, json_response +from botbuilder.core import ( + BotFrameworkAdapterSettings, + TurnContext, + BotFrameworkAdapter, +) +from botbuilder.core.integration import aiohttp_error_middleware +from botbuilder.schema import Activity, ActivityTypes + +from bots import EchoBot +from config import DefaultConfig + +CONFIG = DefaultConfig() + +# Create adapter. +# See https://aka.ms/about-bot-adapter to learn more about how bots work. +SETTINGS = BotFrameworkAdapterSettings(CONFIG.APP_ID, CONFIG.APP_PASSWORD) +ADAPTER = BotFrameworkAdapter(SETTINGS) + + +# Catch-all for errors. +async def on_error(context: TurnContext, error: Exception): + # This check writes out errors to console log .vs. app insights. + # NOTE: In production environment, you should consider logging this to Azure + # application insights. + print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) + traceback.print_exc() + + # Send a message to the user + await context.send_activity("The bot encountered an error or bug.") + await context.send_activity( + "To continue to run this bot, please fix the bot source code." + ) + # Send a trace activity if we're talking to the Bot Framework Emulator + if context.activity.channel_id == "emulator": + # Create a trace activity that contains the error object + trace_activity = Activity( + label="TurnError", + name="on_turn_error Trace", + timestamp=datetime.utcnow(), + type=ActivityTypes.trace, + value=f"{error}", + value_type="https://www.botframework.com/schemas/error", + ) + # Send a trace activity, which will be displayed in Bot Framework Emulator + await context.send_activity(trace_activity) + + +ADAPTER.on_turn_error = on_error + +# Create the Bot +BOT = EchoBot() + + +# Listen for incoming requests on /api/messages +async def messages(req: Request) -> Response: + # Main bot message handler. + if "application/json" in req.headers["Content-Type"]: + body = await req.json() + else: + return Response(status=415) + + activity = Activity().deserialize(body) + auth_header = req.headers["Authorization"] if "Authorization" in req.headers else "" + + response = await ADAPTER.process_activity(activity, auth_header, BOT.on_turn) + if response: + return json_response(data=response.body, status=response.status) + return Response(status=201) + + +APP = web.Application(middlewares=[aiohttp_error_middleware]) +APP.router.add_post("/api/messages", messages) + +if __name__ == "__main__": + try: + web.run_app(APP, host="localhost", port=CONFIG.PORT) + except Exception as error: + raise error diff --git a/samples/experimental/streamming-extensions/bots/__init__.py b/samples/experimental/streamming-extensions/bots/__init__.py new file mode 100644 index 000000000..f95fbbbad --- /dev/null +++ b/samples/experimental/streamming-extensions/bots/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .echo_bot import EchoBot + +__all__ = ["EchoBot"] diff --git a/samples/experimental/streamming-extensions/bots/echo_bot.py b/samples/experimental/streamming-extensions/bots/echo_bot.py new file mode 100644 index 000000000..90a094640 --- /dev/null +++ b/samples/experimental/streamming-extensions/bots/echo_bot.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from botbuilder.core import ActivityHandler, MessageFactory, TurnContext +from botbuilder.schema import ChannelAccount + + +class EchoBot(ActivityHandler): + async def on_members_added_activity( + self, members_added: [ChannelAccount], turn_context: TurnContext + ): + for member in members_added: + if member.id != turn_context.activity.recipient.id: + await turn_context.send_activity("Hello and welcome!") + + async def on_message_activity(self, turn_context: TurnContext): + return await turn_context.send_activity( + MessageFactory.text(f"Echo: {turn_context.activity.text}") + ) diff --git a/samples/experimental/streamming-extensions/config.py b/samples/experimental/streamming-extensions/config.py new file mode 100644 index 000000000..e007d0fa9 --- /dev/null +++ b/samples/experimental/streamming-extensions/config.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os + +""" Bot Configuration """ + + +class DefaultConfig: + """ Bot Configuration """ + + PORT = 3978 + APP_ID = os.environ.get("MicrosoftAppId", "") + APP_PASSWORD = os.environ.get("MicrosoftAppPassword", "") From 62f8a6fc58fedac4fb74ce8951e695343269b98f Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 1 Apr 2020 23:48:41 -0700 Subject: [PATCH 02/37] Updates for POC. This is WIP --- .../streaming/streaming_request_handler.py | 36 ++++++++++- .../botbuilder/streaming/__init__.py | 4 +- .../streaming/payloads/request_manager.py | 39 ++++++++++++ .../botbuilder/streaming/receive_response.py | 10 +++ .../botbuilder/streaming/streaming_request.py | 63 +++++++++++++++++++ .../streaming/streaming_response.py | 3 +- .../streaming/transport/__init__.py | 9 +++ .../transport/streaming_transport_service.py | 12 ++++ .../transport/web_socket/__init__.py | 9 +++ .../transport/web_socket/web_socket.py | 7 +++ .../transport/web_socket/web_socket_server.py | 12 ++++ libraries/botbuilder-streaming/setup.py | 7 ++- 12 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index 5efb2ffa5..bb16ceade 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -1,8 +1,38 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from botbuilder.core import bot +import platform +from datetime import datetime +from logging import Logger +from typing import Dict +from botbuilder.core import Bot +from botbuilder.streaming import RequestHandler, __title__, __version__ +from botbuilder.streaming.transport.web_socket import WebSocket -class StreamingRequestHandler: - pass +from .streaming_activity_processor import StreamingActivityProcessor + + +class StreamingRequestHandler(RequestHandler): + def __init__(self, bot: Bot, activity_processor: StreamingActivityProcessor, web_socket: WebSocket, logger: Logger = None): + if not bot: + raise TypeError("'bot' argument can not be None") + if not activity_processor: + raise TypeError("'activity_processor' argument can not be None") + + self._bot = bot + self._activity_processor = activity_processor + self._logger = logger + self._conversations: Dict[str, datetime] = {} + self._user_agent = StreamingRequestHandler._get_user_agent() + self._server = + + @staticmethod + def _get_user_agent() -> str: + package_user_agent = f"{__title__}/{__version__}" + uname = platform.uname() + os_version = f"{uname.machine}-{uname.system}-{uname.version}" + py_version = f"Python,Version={platform.python_version()}" + platform_user_agent = f"({os_version}; {py_version})" + user_agent = f"{package_user_agent} {platform_user_agent}" + return user_agent diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index 6acbe2231..06cbc2dbf 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -5,8 +5,10 @@ # license information. # -------------------------------------------------------------------------- +from .about import __version__, __title__ from .receive_request import ReceiveRequest +from .receive_response import ReceiveResponse from .request_handler import RequestHandler from .streaming_response import StreamingResponse -__all__ = ["ReceiveRequest", "RequestHandler", "StreamingResponse"] +__all__ = ["ReceiveRequest", "ReceiveResponse", "RequestHandler", "StreamingResponse", "__title__", "__version__"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py new file mode 100644 index 000000000..9bdf47275 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from asyncio import Future +from uuid import UUID +from typing import Dict + +from botbuilder.streaming import ReceiveResponse + + +class RequestManager: + def __init__(self, *, pending_requests: Dict[UUID, "Future[ReceiveResponse]"]): + self._pending_requests = pending_requests or {} + + def signal_response(self, request_id: UUID, response: "Future[ReceiveResponse]") -> bool: + #TODO: dive more into this logic + signal: Future = self._pending_requests.get(request_id) + if signal: + signal.set_result(response) + #TODO: double check this + del self._pending_requests[request_id] + + return True + + return False + + async def get_response(self, request_id: UUID) -> ReceiveResponse: + if request_id in self._pending_requests: + return None + + pending_request = Future() + self._pending_requests[request_id] = pending_request + + try: + response: ReceiveResponse = await pending_request + return response + + finally: + del self._pending_requests[request_id] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py new file mode 100644 index 000000000..e1831cfa0 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + + +class ReceiveResponse: + def __init__(self, status_code: int = None, streams: List[object] = None): + self.status_code = status_code + self.streams = streams diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py new file mode 100644 index 000000000..8f1612357 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID, uuid4 +from typing import List + +from botbuilder.streaming.payloads import ResponseMessageStream + + +class StreamingRequest: + GET = "GET" + POST = "POST" + PUT = "PUT" + DELETE = "DELETE" + + def __init__(self, *, verb: str = None, path: str = None, streams: List[ResponseMessageStream] = None): + self.verb = verb + self.path = path + self.streams = streams + + @staticmethod + def create_request(method: str, path: str = None, body: object = None) -> "StreamingRequest": + if not method: + return None + + request = StreamingRequest( + verb=method, + path=path, + ) + + if body: + request.add_stream(body) + + return request + + @staticmethod + def create_get(path: str = None, body: object = None) -> "StreamingRequest": + return StreamingRequest.create_request("GET", path, body) + + @staticmethod + def create_post(path: str = None, body: object = None) -> "StreamingRequest": + return StreamingRequest.create_request("POST", path, body) + + @staticmethod + def create_put(path: str = None, body: object = None) -> "StreamingRequest": + return StreamingRequest.create_request("PUT", path, body) + + @staticmethod + def create_delete(path: str = None, body: object = None) -> "StreamingRequest": + return StreamingRequest.create_request("DELETE", path, body) + + def add_stream(self, content: object, stream_id: UUID = None): + if not content: + raise TypeError("'content' argument can not be None") + if not self.streams: + self.streams = [] + + self.streams.append( + ResponseMessageStream( + id=stream_id or uuid4(), + content=content + ) + ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py index e23187860..8de0fb31b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py @@ -3,8 +3,7 @@ from typing import List -# TODO: reconsider to absolute import -from .payloads import ResponseMessageStream +from botbuilder.streaming.payloads import ResponseMessageStream class StreamingResponse: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py new file mode 100644 index 000000000..5d07e4b01 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from .streaming_transport_service import StreamingTransportService + +__all__ = [ + "StreamingTransportService" +] \ No newline at end of file diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py new file mode 100644 index 000000000..498f7198c --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC + + +class StreamingTransportService(ABC): + async def start(self): + raise NotImplementedError() + + async def send(self, request): + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py new file mode 100644 index 000000000..b6f2e1d57 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from .web_socket import WebSocket + +__all__ = [ + "WebSocket" +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py new file mode 100644 index 000000000..3b67ac1f7 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +class WebSocket: + def __init__(self): + self diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py new file mode 100644 index 000000000..c234f577a --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from botbuilder.streaming import RequestHandler + +from .web_socket import WebSocket + + +class WebSocketServer: + def __init__(self, socket: WebSocket, request_handler: RequestHandler): + self._request_handler = request_handler + diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botbuilder-streaming/setup.py index 0a5d36fbd..639a0f444 100644 --- a/libraries/botbuilder-streaming/setup.py +++ b/libraries/botbuilder-streaming/setup.py @@ -31,7 +31,12 @@ long_description=long_description, long_description_content_type="text/x-rst", license=package_info["__license__"], - packages=["botbuilder.streaming", "botbuilder.streaming.payloads"], + packages=[ + "botbuilder.streaming", + "botbuilder.streaming.payloads", + "botbuilder.streaming.transport", + "botbuilder.streaming.transport.web_socket" + ], install_requires=REQUIRES, classifiers=[ "Programming Language :: Python :: 3.7", From 8be271c88de823a584322c2538616612d2c6fda5 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 1 Apr 2020 23:49:39 -0700 Subject: [PATCH 03/37] Pylint: POC updates --- .../streaming/streaming_request_handler.py | 10 +++++++-- .../botbuilder/streaming/__init__.py | 9 +++++++- .../streaming/payloads/request_manager.py | 10 +++++---- .../botbuilder/streaming/streaming_request.py | 22 ++++++++++--------- .../streaming/transport/__init__.py | 4 +--- .../transport/web_socket/__init__.py | 4 +--- .../transport/web_socket/web_socket_server.py | 1 - libraries/botbuilder-streaming/setup.py | 2 +- 8 files changed, 37 insertions(+), 25 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index bb16ceade..a8c7efeaf 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -14,7 +14,13 @@ class StreamingRequestHandler(RequestHandler): - def __init__(self, bot: Bot, activity_processor: StreamingActivityProcessor, web_socket: WebSocket, logger: Logger = None): + def __init__( + self, + bot: Bot, + activity_processor: StreamingActivityProcessor, + web_socket: WebSocket, + logger: Logger = None, + ): if not bot: raise TypeError("'bot' argument can not be None") if not activity_processor: @@ -25,7 +31,7 @@ def __init__(self, bot: Bot, activity_processor: StreamingActivityProcessor, web self._logger = logger self._conversations: Dict[str, datetime] = {} self._user_agent = StreamingRequestHandler._get_user_agent() - self._server = + self._server @staticmethod def _get_user_agent() -> str: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index 06cbc2dbf..e4f0eaa9c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -11,4 +11,11 @@ from .request_handler import RequestHandler from .streaming_response import StreamingResponse -__all__ = ["ReceiveRequest", "ReceiveResponse", "RequestHandler", "StreamingResponse", "__title__", "__version__"] +__all__ = [ + "ReceiveRequest", + "ReceiveResponse", + "RequestHandler", + "StreamingResponse", + "__title__", + "__version__", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py index 9bdf47275..2f3064969 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py @@ -12,18 +12,20 @@ class RequestManager: def __init__(self, *, pending_requests: Dict[UUID, "Future[ReceiveResponse]"]): self._pending_requests = pending_requests or {} - def signal_response(self, request_id: UUID, response: "Future[ReceiveResponse]") -> bool: - #TODO: dive more into this logic + def signal_response( + self, request_id: UUID, response: "Future[ReceiveResponse]" + ) -> bool: + # TODO: dive more into this logic signal: Future = self._pending_requests.get(request_id) if signal: signal.set_result(response) - #TODO: double check this + # TODO: double check this del self._pending_requests[request_id] return True return False - + async def get_response(self, request_id: UUID) -> ReceiveResponse: if request_id in self._pending_requests: return None diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py index 8f1612357..cded490ca 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -13,20 +13,25 @@ class StreamingRequest: PUT = "PUT" DELETE = "DELETE" - def __init__(self, *, verb: str = None, path: str = None, streams: List[ResponseMessageStream] = None): + def __init__( + self, + *, + verb: str = None, + path: str = None, + streams: List[ResponseMessageStream] = None + ): self.verb = verb self.path = path self.streams = streams @staticmethod - def create_request(method: str, path: str = None, body: object = None) -> "StreamingRequest": + def create_request( + method: str, path: str = None, body: object = None + ) -> "StreamingRequest": if not method: return None - request = StreamingRequest( - verb=method, - path=path, - ) + request = StreamingRequest(verb=method, path=path,) if body: request.add_stream(body) @@ -56,8 +61,5 @@ def add_stream(self, content: object, stream_id: UUID = None): self.streams = [] self.streams.append( - ResponseMessageStream( - id=stream_id or uuid4(), - content=content - ) + ResponseMessageStream(id=stream_id or uuid4(), content=content) ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py index 5d07e4b01..96e516cca 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py @@ -4,6 +4,4 @@ from .streaming_transport_service import StreamingTransportService -__all__ = [ - "StreamingTransportService" -] \ No newline at end of file +__all__ = ["StreamingTransportService"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py index b6f2e1d57..d3ce66780 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py @@ -4,6 +4,4 @@ from .web_socket import WebSocket -__all__ = [ - "WebSocket" -] +__all__ = ["WebSocket"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index c234f577a..b5bfcbfaf 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -9,4 +9,3 @@ class WebSocketServer: def __init__(self, socket: WebSocket, request_handler: RequestHandler): self._request_handler = request_handler - diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botbuilder-streaming/setup.py index 639a0f444..357fa1431 100644 --- a/libraries/botbuilder-streaming/setup.py +++ b/libraries/botbuilder-streaming/setup.py @@ -35,7 +35,7 @@ "botbuilder.streaming", "botbuilder.streaming.payloads", "botbuilder.streaming.transport", - "botbuilder.streaming.transport.web_socket" + "botbuilder.streaming.transport.web_socket", ], install_requires=REQUIRES, classifiers=[ From bd4f69194c7c567328dfa0b4de3eb5e91b576b77 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Mon, 6 Apr 2020 09:20:25 -0700 Subject: [PATCH 04/37] Updates on POC, protocol adapter pending --- .../streaming/streaming_request_handler.py | 6 +- .../streaming/payload_transport/__init__.py | 10 ++ .../payload_transport/payload_receiver.py | 142 ++++++++++++++++ .../payload_transport/payload_sender.py | 149 +++++++++++++++++ .../payload_transport/send_packet.py | 21 +++ .../streaming/payload_transport/send_queue.py | 44 +++++ .../botbuilder/streaming/payloads/__init__.py | 4 +- .../streaming/payloads/header_serializer.py | 156 ++++++++++++++++++ .../streaming/payloads/models/__init__.py | 8 + .../streaming/payloads/models/header.py | 32 ++++ .../payloads/models/payload_types.py | 16 ++ .../botbuilder/streaming/streaming_request.py | 6 +- .../streaming/transport/__init__.py | 15 +- .../transport/disconnected_event_args.py | 10 ++ .../streaming/transport/transport_base.py | 10 ++ .../transport/transport_constants.py | 11 ++ .../transport/transport_receiver_base.py | 11 ++ .../transport/transport_sender_base.py | 11 ++ .../transport/web_socket/__init__.py | 14 +- .../web_socket/web_socket_close_status.py | 17 ++ .../web_socket/web_socket_message_type.py | 14 ++ .../transport/web_socket/web_socket_server.py | 18 ++ .../transport/web_socket/web_socket_state.py | 9 + .../web_socket/web_socket_transport.py | 77 +++++++++ libraries/botbuilder-streaming/setup.py | 2 + 25 files changed, 805 insertions(+), 8 deletions(-) create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index a8c7efeaf..a1dd667a6 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -22,9 +22,11 @@ def __init__( logger: Logger = None, ): if not bot: - raise TypeError("'bot' argument can not be None") + raise TypeError(f"'bot: {bot.__class__.__name__}' argument can't be None") if not activity_processor: - raise TypeError("'activity_processor' argument can not be None") + raise TypeError( + f"'activity_processor: {activity_processor.__class__.__name__}' argument can't be None" + ) self._bot = bot self._activity_processor = activity_processor diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py new file mode 100644 index 000000000..6270c96f3 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from .payload_receiver import PayloadReceiver +from .payload_sender import PayloadSender +from .send_packet import SendPacket + + +__all__ = ["PayloadReceiver", "PayloadSender", "SendPacket"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py new file mode 100644 index 000000000..27c82f128 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from typing import Callable, List + +from botbuilder.streaming.payloads import HeaderSerializer +from botbuilder.streaming.payloads.models import Header, PayloadTypes +from botbuilder.streaming.transport import ( + DisconnectedEventArgs, + TransportConstants, + TransportReceiverBase, +) + + +class PayloadReceiver: + def __init__(self): + self._get_stream: Callable[[Header], List[int]] = None + self._receive_action: Callable[[Header, List[int], int], None] = None + self._receiver: TransportReceiverBase = None + self._is_disconnecting = False + + self._receive_header_buffer: List[int] = [ + None + ] * TransportConstants.MAX_HEADER_LENGTH + self._receive_content_buffer: List[int] = [ + None + ] * TransportConstants.MAX_PAYLOAD_LENGTH + + self.disconnected: Callable[[object, DisconnectedEventArgs], None] = None + + @property + def is_connected(self) -> bool: + return self._receiver is None + + def connect(self, receiver: TransportReceiverBase): + if self._receiver: + raise RuntimeError(f"{self.__class__.__name__} instance already connected.") + + self._receiver = receiver + self._connected_event.set() + + def _run_receive(self): + asyncio.create_task(self._receive_packets()) + + def disconnect(self, event_args: DisconnectedEventArgs = None): + did_disconnect = False + + if not self._is_disconnecting: + self._is_disconnecting = True + try: + try: + if self._receiver: + self._receiver.close() + # TODO: investigate if 'dispose' is necessary + did_disconnect = True + except Exception: + pass + + self._receiver = None + + if did_disconnect: + if self.disconnected: + self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) + finally: + self._is_disconnecting = False + + async def _receive_packets(self): + is_closed = False + + while self._receiver and self._receiver.is_connected and not is_closed: + # receive a single packet + try: + # read the header + header_offset = 0 + while header_offset < TransportConstants.MAX_HEADER_LENGTH: + length = await self._receiver.receive( + self._receive_header_buffer, + header_offset, + TransportConstants.MAX_HEADER_LENGTH - header_offset, + ) + + if length == 0: + # TODO: make custom exception + raise Exception( + "TransportDisconnectedException: Stream closed while reading header bytes" + ) + + header_offset += length + + # deserialize the bytes into a header + header = HeaderSerializer.deserialize( + self._receive_header_buffer, 0, TransportConstants.MAX_HEADER_LENGTH + ) + + # read the payload + content_stream = self._get_stream(header) + + buffer = ( + [None] * header.payload_length + if PayloadTypes.is_stream(header) + else self._receive_content_buffer + ) + offset = 0 + + if header.payload_length: + while offset < header.payload_length: + count = min( + header.payload_length - offset, + TransportConstants.MAX_PAYLOAD_LENGTH, + ) + + # Send: Packet content + length = await self._receiver.receive(buffer, offset, count) + if length == 0: + # TODO: make custom exception + raise Exception( + "TransportDisconnectedException: Stream closed while reading header bytes" + ) + + if content_stream: + # write chunks to the content_stream if it's not a stream type + # TODO: this has to be improved in custom buffer class (validate buffer ended) + if not PayloadTypes.is_stream(header): + for index in range(offset, offset + length): + content_stream[index] = buffer[index] + + offset += length + + # give the full payload buffer to the contentStream if it's a stream + if content_stream and PayloadTypes.is_stream(header): + # TODO: should this be a copy? + content_stream = buffer + + self._receive_action(header, content_stream, offset) + except Exception as exception: + is_closed = True + disconnect_args = DisconnectedEventArgs(reason=exception.message) + + self.disconnect(disconnect_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py new file mode 100644 index 000000000..3f9ef399e --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -0,0 +1,149 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from threading import Event +from typing import Awaitable, Callable, List + +from botbuilder.streaming.transport import ( + DisconnectedEventArgs, + TransportSenderBase, + TransportConstants, +) +from botbuilder.streaming.payloads import HeaderSerializer +from botbuilder.streaming.payloads.models import Header + +from .send_queue import SendQueue +from .send_packet import SendPacket + + +# TODO: consider interface this class +class PayloadSender: + def __init__(self,): + self._connected_event = Event() + self._sender: TransportSenderBase = None + self._is_disconnecting: bool = False + self._send_header_buffer: List[int] = [ + None + ] * TransportConstants.MAX_HEADER_LENGTH + self._send_content_buffer: List[int] = [ + None + ] * TransportConstants.MAX_PAYLOAD_LENGTH + + self._send_queue = SendQueue(action=self._write_packet) + + self.disconnected: Callable[[object, DisconnectedEventArgs], None] = None + + @property + def is_connected(self) -> bool: + return self._sender is None + + def connect(self, sender: TransportSenderBase): + if self._sender: + raise RuntimeError(f"{self.__class__.__name__} instance already connected.") + + self._sender = sender + self._connected_event.set() + + # TODO: check 'stream' for payload + def send_payload( + self, + header: Header, + payload: object, + is_length_known: bool, + sent_callback: Callable[[Header], Awaitable], + ): + packet = SendPacket( + header=header, + payload=payload, + is_length_known=is_length_known, + sent_callback=sent_callback, + ) + + self._send_queue.post(packet) + + def disconnect(self, event_args: DisconnectedEventArgs = None): + did_disconnect = False + + if not self._is_disconnecting: + self._is_disconnecting = True + try: + try: + if self._sender: + self._sender.close() + # TODO: investigate if 'dispose' is necessary + did_disconnect = True + except Exception: + pass + + self._sender = None + + if did_disconnect: + self._connected_event.clear() + if self.disconnected: + self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) + finally: + self._is_disconnecting = False + + async def _write_packet(self, packet: SendPacket): + self._connected_event.wait() + + try: + # determine if we know the payload length and end + if not packet.is_length_known: + count = packet.header.payload_length + packet.header.end = count == 0 + + header_length = HeaderSerializer.serialize( + packet.header, self._send_header_buffer, 0 + ) + + # Send: Packet Header + length = await self._sender.send(self._send_header_buffer, 0, header_length) + if not length: + # TODO: make custom exception + raise Exception("TransportDisconnectedException") + + offset = 0 + + # Send content in chunks + if packet.header.payload_length and packet.payload: + # If we already read the buffer, send that + # If we did not, read from the stream until we've sent that amount + if not packet.is_length_known: + # Send: Packet content + length = await self._sender.send( + self._send_content_buffer, 0, packet.header.payload_length + ) + if length == 0: + # TODO: make custom exception + raise Exception("TransportDisconnectedException") + else: + while offset < packet.header.payload_length: + count = min( + packet.header.payload_length - offset, + TransportConstants.MAX_PAYLOAD_LENGTH, + ) + + # copy the stream to the buffer + # TODO: this has to be improved in custom buffer class (validate buffer ended) + for index in range(count): + self._send_content_buffer[index] = packet.payload[index] + count = len(self._send_content_buffer) + + # Send: Packet content + length = await self._sender.send( + self._send_content_buffer, 0, count + ) + if length == 0: + # TODO: make custom exception + raise Exception("TransportDisconnectedException") + + if packet.sent_callback: + # TODO: should this really run in the background? + asyncio.create_task(packet.sent_callback(packet.header)) + except Exception as exception: + disconnected_args = DisconnectedEventArgs(reason=exception.message) + self.disconnect(disconnected_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py new file mode 100644 index 000000000..08e4a770e --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Awaitable, Callable + +from botbuilder.streaming.payloads.models import Header + + +class SendPacket: + def __init__( + self, + *, + header: Header, + payload: object, + is_length_known: bool, + sent_callback: Callable[[Header], Awaitable] + ): + self.header = header + self.payload = payload + self.is_length_known = is_length_known + self.sent_callback = sent_callback diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py new file mode 100644 index 000000000..e089d93f5 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from queue import Queue +from typing import Awaitable, Callable +from threading import Event, Lock, Semaphore + +import asyncio + + +class SendQueue: + def __init__(self, action: Callable[[object], Awaitable], timeout: int = 30): + self._action = action + + self._queue = Queue() + self._timeout_seconds = timeout + + # TODO: this have to be abstracted so can remove asyncio dependency + loop = asyncio.get_event_loop() + loop.create_task(self._process) + + def post(self, item: object): + self.post_internal(item) + + def post_internal(self, item: object): + self._queue.put(item) + + async def process(self): + while True: + try: + while True: + item = self._queue.get() + if not item: + break + try: + await self._action(item) + except Exception: + # AppInsights.TrackException(e) + pass + finally: + self._queue.task_done() + except Exception: + # AppInsights.TrackException(e) + pass diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py index e964074d9..2bcb92561 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from .header_serializer import HeaderSerializer +from .request_manager import RequestManager from .response_message_stream import ResponseMessageStream -__all__ = ["ResponseMessageStream"] +__all__ = ["RequestManager", "ResponseMessageStream", "HeaderSerializer"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py new file mode 100644 index 000000000..faa0954e3 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import List + +from botbuilder.streaming.transport import TransportConstants + +from .models import Header + +_char_to_binary_int = {val.decode(): list(val)[0] for val in [b".", b"\n", b"1", b"0"]} + + +# TODO: consider abstracting the binary int list logic into a class for easier handling +class HeaderSerializer: + DELIMITER = _char_to_binary_int["."] + TERMINATOR = _char_to_binary_int["\n"] + END = _char_to_binary_int["1"] + NOT_END = _char_to_binary_int["0"] + TYPE_OFFSET = 0 + TYPE_DELIMITER_OFFSET = 1 + LENGTH_OFFSET = 2 + LENGTH_LENGTH = 6 + LENGTH_DELIMETER_OFFSET = 8 + ID_OFFSET = 9 + ID_LENGTH = 36 + ID_DELIMETER_OFFSET = 45 + END_OFFSET = 46 + TERMINATOR_OFFSET = 47 + + @staticmethod + def serialize(header: Header, buffer: List[int], offset: int) -> int: + + # write type + buffer[HeaderSerializer.TYPE_OFFSET] = HeaderSerializer._char_to_binary_int( + header.type + ) + buffer[HeaderSerializer.TYPE_DELIMITER_OFFSET] = HeaderSerializer.DELIMITER + + # write length + length_binary_array: List[int] = list( + HeaderSerializer._int_to_formatted_encoded_str( + header.payload_length, "{:06d}" + ) + ) + HeaderSerializer._write_in_buffer( + length_binary_array, buffer, HeaderSerializer.LENGTH_OFFSET + ) + buffer[HeaderSerializer.LENGTH_DELIMETER_OFFSET] = HeaderSerializer.DELIMITER + + # write id + id_binary_array: List[int] = list( + HeaderSerializer._uuid_to_numeric_encoded_str(header.id) + ) + HeaderSerializer._write_in_buffer( + id_binary_array, buffer, HeaderSerializer.ID_OFFSET + ) + buffer[HeaderSerializer.ID_DELIMETER_OFFSET] = HeaderSerializer.DELIMITER + + # write terminator + buffer[HeaderSerializer.END_OFFSET] = ( + HeaderSerializer.END if header.end else HeaderSerializer.NOT_END + ) + buffer[HeaderSerializer.TERMINATOR_OFFSET] = HeaderSerializer.DELIMITER + + return TransportConstants.MAX_HEADER_LENGTH + + @staticmethod + def deserialize(buffer: List[int], offset: int, count: int) -> Header: + if count != TransportConstants.MAX_HEADER_LENGTH: + raise ValueError("Cannot deserialize header, incorrect length") + + header = Header( + type=HeaderSerializer._binary_int_to_char( + buffer[HeaderSerializer.TYPE_OFFSET] + ) + ) + + if buffer[HeaderSerializer.TYPE_DELIMITER_OFFSET] != HeaderSerializer.DELIMITER: + raise ValueError("Header type delimeter is malformed") + + length_str = HeaderSerializer._binary_array_to_str( + buffer[HeaderSerializer.LENGTH_OFFSET : HeaderSerializer.LENGTH_LENGTH] + ) + + try: + length = int(length_str) + except Exception: + raise ValueError("Header length is malformed") + + header.payload_length = length + + if ( + buffer[HeaderSerializer.LENGTH_DELIMETER_OFFSET] + != HeaderSerializer.DELIMITER + ): + raise ValueError("Header length delimeter is malformed") + + identifier_str = HeaderSerializer._binary_array_to_str( + buffer[HeaderSerializer.ID_OFFSET : HeaderSerializer.ID_LENGTH] + ) + + try: + identifier = UUID(int=int(identifier_str)) + except Exception: + raise ValueError("Header id is malformed") + + header.id = identifier + + if buffer[HeaderSerializer.ID_DELIMETER_OFFSET] != HeaderSerializer.DELIMITER: + raise ValueError("Header id delimeter is malformed") + + if buffer[HeaderSerializer.END_OFFSET] not in [ + HeaderSerializer.END, + HeaderSerializer.NOT_END, + ]: + raise ValueError("Header end is malformed") + + if buffer[HeaderSerializer.TERMINATOR_OFFSET] != HeaderSerializer.TERMINATOR: + raise ValueError("Header terminator is malformed") + + return header + + @staticmethod + def _char_to_binary_int(char: str) -> int: + if len(char) != 1: + raise ValueError("Char to cast should be a str of exactly length 1") + + unicode_list = list(char.encode()) + + if len(unicode_list) != 1: + raise ValueError("Char to cast should be in the ASCII domain") + + return unicode_list[0] + + @staticmethod + def _int_to_formatted_encoded_str(value: int, str_format: str) -> bytes: + return str_format.format(value).encode() + + @staticmethod + def _uuid_to_numeric_encoded_str(value: UUID) -> bytes: + return str(int(value)).encode() + + @staticmethod + def _binary_int_to_char(binary_int: int) -> str: + return bytes([binary_int]).decode("utf8") + + @staticmethod + def _binary_array_to_str(binary_array: List[int]) -> str: + return bytes(binary_array).decode("utf8") + + @staticmethod + def _write_in_buffer(data: List[int], buffer: List[int], insert_index: int): + for byte_int in data: + buffer[insert_index] = byte_int + insert_index += 1 diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py new file mode 100644 index 000000000..f582bd0f5 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from .header import Header +from .payload_types import PayloadTypes + +__all__ = ["Header", "PayloadTypes"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py new file mode 100644 index 000000000..fd9e24d88 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID + +from botbuilder.streaming.transport import TransportConstants + + +class Header: + def __init__(self, *, type: str = None, id: UUID = None, end: bool = None): + self._internal_payload_length = None + self.type: str = type + self.id: UUID = id + self.end: bool = end + + @property + def payload_length(self) -> int: + return self._internal_payload_length + + @payload_length.setter + def payload_length(self, value: int): + self._validate_length( + value, TransportConstants.MAX_LENGTH, TransportConstants.MIN_LENGTH + ) + self._internal_payload_length = value + + def _validate_length(self, value: int, max_val: int, min_val: int): + if value > max: + raise ValueError(f"Length must be less or equal than {max_val}") + + if value < min: + raise ValueError(f"Length must be greater or equal than {min_val}") diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py new file mode 100644 index 000000000..ec9c01090 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .header import Header + + +class PayloadTypes: + REQUEST = "A" + RESPONSE = "B" + STREAM = "S" + CANCEL_ALL = "X" + CANCEL_STREAM = "C" + + @staticmethod + def is_stream(header: Header) -> bool: + return header.type == PayloadTypes.STREAM diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py index cded490ca..1d3d5f319 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -18,7 +18,7 @@ def __init__( *, verb: str = None, path: str = None, - streams: List[ResponseMessageStream] = None + streams: List[ResponseMessageStream] = None, ): self.verb = verb self.path = path @@ -56,7 +56,9 @@ def create_delete(path: str = None, body: object = None) -> "StreamingRequest": def add_stream(self, content: object, stream_id: UUID = None): if not content: - raise TypeError("'content' argument can not be None") + raise TypeError( + f"'content: {content.__class__.__name__}' argument can't be None" + ) if not self.streams: self.streams = [] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py index 96e516cca..3939e47a5 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py @@ -1,7 +1,18 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +from .disconnected_event_args import DisconnectedEventArgs from .streaming_transport_service import StreamingTransportService +from .transport_base import TransportBase +from .transport_constants import TransportConstants +from .transport_receiver_base import TransportReceiverBase +from .transport_sender_base import TransportSenderBase -__all__ = ["StreamingTransportService"] +__all__ = [ + "DisconnectedEventArgs", + "StreamingTransportService", + "TransportBase", + "TransportConstants", + "TransportReceiverBase", + "TransportSenderBase", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py new file mode 100644 index 000000000..9db882219 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +class DisconnectedEventArgs: + def __init__(self, *, reason: str = None): + self.reason = reason + + +DisconnectedEventArgs.empty = DisconnectedEventArgs() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py new file mode 100644 index 000000000..4955f96e8 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +class TransportBase: + def __init__(self): + self.is_connected: bool = None + + def close(self): + return diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py new file mode 100644 index 000000000..139099512 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC + + +class TransportConstants(ABC): + MAX_PAYLOAD_LENGTH = 4096 + MAX_HEADER_LENGTH = 48 + MAX_LENGTH = 999999 + MIN_LENGTH = 0 diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py new file mode 100644 index 000000000..e7e849a49 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC + +from .transport_base import TransportBase + + +class TransportReceiverBase(ABC, TransportBase): + async def receive(self, buffer: object, offset: int, count: int) -> int: + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py new file mode 100644 index 000000000..33d647159 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from abc import ABC + +from .transport_base import TransportBase + + +class TransportSenderBase(ABC, TransportBase): + async def send(self, buffer: object, offset: int, count: int) -> int: + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py index d3ce66780..335aa0808 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py @@ -3,5 +3,17 @@ from .web_socket import WebSocket +from .web_socket_close_status import WebSocketCloseStatus +from .web_socket_server import WebSocketServer +from .web_socket_message_type import WebSocketMessageType +from .web_socket_transport import WebSocketTransport +from .web_socket_state import WebSocketState -__all__ = ["WebSocket"] +__all__ = [ + "WebSocket", + "WebSocketCloseStatus", + "WebSocketMessageType", + "WebSocketServer", + "WebSocketTransport", + "WebSocketState", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py new file mode 100644 index 000000000..417c6588c --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from enum import IntEnum + + +class WebSocketCloseStatus(IntEnum): + NORMAL_CLOSURE = 1000 + ENDPOINT_UNAVAILABLE = 1001 + PROTOCOL_ERROR = 1002 + INVALID_MESSAGE_TYPE = 1003 + EMPTY = 1005 + INVALID_PAYLOAD_DATA = 1007 + POLICY_VIOLATION = 1008 + MESSAGE_TOO_BIG = 1009 + MANDATORY_EXTENSION = 1010 + INTERNAL_SERVER_ERROR = 1011 diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py new file mode 100644 index 000000000..658b7e073 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from enum import IntEnum + + +class WebSocketMessageType(IntEnum): + # websocket spec types + CONTINUATION = 0 + TEXT = 1 + BINARY = 2 + PING = 9 + PONG = 10 + CLOSE = 8 diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index b5bfcbfaf..37bbff37b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -2,10 +2,28 @@ # Licensed under the MIT License. from botbuilder.streaming import RequestHandler +from botbuilder.streaming.payloads import RequestManager +from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver from .web_socket import WebSocket +from .web_socket_transport import WebSocketTransport class WebSocketServer: def __init__(self, socket: WebSocket, request_handler: RequestHandler): + if not socket: + raise TypeError( + f"'socket: {socket.__class__.__name__}' argument can't be None" + ) + if not request_handler: + raise TypeError( + f"'request_handler: {request_handler.__class__.__name__}' argument can't be None" + ) + + self._web_socket_transport = WebSocketTransport(socket) self._request_handler = request_handler + self._request_manager = RequestManager() + self._sender = PayloadSender() + self._sender.disconnected = self._on_connection_disconnected + self._receiver = PayloadReceiver() + self._receiver.disconnected = self._on_connection_disconnected diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py new file mode 100644 index 000000000..fddd42ec2 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from enum import IntEnum + + +class WebSocketState(IntEnum): + OPEN = 2 + CLOSED = 5 diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py new file mode 100644 index 000000000..c8bdd4696 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .web_socket import WebSocket +from .web_socket_message_type import WebSocketMessageType +from .web_socket_close_status import WebSocketCloseStatus +from .web_socket_state import WebSocketState + + +class WebSocketTransport: + def __init__(self, web_socket: WebSocket): + self._socket = web_socket + + @property + def is_connected(self): + print("Getting value") + # TODO: mock logic + return self._socket.status == "Open" + + async def close(self): + # TODO: mock logic + if self._socket.status == "Open": + try: + await self._socket.close( + WebSocketCloseStatus.NORMAL_CLOSURE, + "Closed by the WebSocketTransport", + ) + except Exception: + """ + Any exception thrown here will be caused by the socket already being closed, + which is the state we want to put it in by calling this method, which + means we don't care if it was already closed and threw an exception + when we tried to close it again. + """ + pass + + # TODO: might need to remove offset and count if no segmentation possible + # TODO: considering to create a BFTransportBuffer class to abstract the logic of binary buffers adapting to + # current interfaces + async def receive( + self, buffer: [object], offset: int = None, count: int = None + ) -> int: + try: + if self._socket: + result = await self._socket.receive() + buffer.append(result) + if result.message_type == WebSocketMessageType.CLOSE: + await self._socket.close( + WebSocketCloseStatus.NORMAL_CLOSURE, "Socket closed" + ) + + # Depending on ws implementation library next line might not be necessary + if self._socket.status == WebSocketState.CLOSED: + self._socket.dispose() + + return len(result) + except Exception as error: + # Exceptions of the three types below will also have set the socket's state to closed, which fires an + # event consumers of this class are subscribed to and have handling around. Any other exception needs to + # be thrown to cause a non-transport-connectivity failure. + raise error + + # TODO: might need to remove offset and count if no segmentation possible (or put them in BFTransportBuffer) + async def send( + self, buffer: [object], offset: int = None, count: int = None + ) -> int: + try: + if self._socket: + await self._socket.send(buffer, WebSocketMessageType.BINARY, True) + return count or len(buffer) + except Exception as error: + # Exceptions of the three types below will also have set the socket's state to closed, which fires an + # event consumers of this class are subscribed to and have handling around. Any other exception needs to + # be thrown to cause a non-transport-connectivity failure. + raise error + + return 0 diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botbuilder-streaming/setup.py index 357fa1431..05eb2f679 100644 --- a/libraries/botbuilder-streaming/setup.py +++ b/libraries/botbuilder-streaming/setup.py @@ -34,6 +34,8 @@ packages=[ "botbuilder.streaming", "botbuilder.streaming.payloads", + "botbuilder.streaming.payloads.models", + "botbuilder.streaming.payload_transport", "botbuilder.streaming.transport", "botbuilder.streaming.transport.web_socket", ], From 95f250f9bedd4cd8efe55f404c58b15dfbf12992 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 7 Apr 2020 00:09:04 -0700 Subject: [PATCH 05/37] Updates on POC, protocol adapter in progress --- .../botbuilder/streaming/__init__.py | 2 + .../payloads/disassemblers/__init__.py | 8 ++ .../disassemblers/payload_disassembler.py | 98 +++++++++++++++++++ .../disassemblers/request_disassembler.py | 40 ++++++++ .../streaming/payloads/models/__init__.py | 11 ++- .../payloads/models/request_payload.py | 42 ++++++++ .../streaming/payloads/models/serializable.py | 14 +++ .../payloads/models/stream_description.py | 33 +++++++ .../streaming/payloads/send_operations.py | 15 +++ .../botbuilder/streaming/protocol_adapter.py | 23 +++++ 10 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index e4f0eaa9c..47fcd2269 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -9,12 +9,14 @@ from .receive_request import ReceiveRequest from .receive_response import ReceiveResponse from .request_handler import RequestHandler +from .streaming_request import StreamingRequest from .streaming_response import StreamingResponse __all__ = [ "ReceiveRequest", "ReceiveResponse", "RequestHandler", + "StreamingRequest", "StreamingResponse", "__title__", "__version__", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py new file mode 100644 index 000000000..2fe75682d --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from .payload_disassembler import PayloadDisassembler +from .request_disassembler import RequestDisassembler + +__all__ = ["PayloadDisassembler", "RequestDisassembler"] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py new file mode 100644 index 000000000..25e316e9a --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from asyncio import Future +from abc import ABC, abstractmethod +from uuid import UUID +from typing import List + +from botbuilder.streaming.transport import TransportConstants +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads import ResponseMessageStream +from botbuilder.streaming.payloads.models import Header, Serializable, StreamDescription + + +class PayloadDisassembler(ABC): + def __init__(self, sender: PayloadSender, identifier: UUID): + self.sender = sender + self.identifier = identifier + self._task_completion_source = Future() + + self._stream: List[int] = None + self._stream_length: int = None + self._send_offset: int = None + self._is_end: bool = False + self._type: str = None + + @property + @abstractmethod + def type(self) -> str: + return self._type + + async def get_stream(self) -> List[int]: + raise NotImplementedError() + + async def disassemble(self): + self._stream = await self.get_stream() + self._stream_length = len(self._stream) + self._send_offset = 0 + + await self._send() + + @staticmethod + def get_stream_description(stream: ResponseMessageStream) -> StreamDescription: + description = StreamDescription(id=str(int(stream.id))) + + # TODO: validate statement below, also make the string a constant + content_type: List[str] = stream.content.headers().get("Content-Type") + if content_type: + description.content_type = content_type[0] + + # TODO: validate statement below, also make the string a constant + content_length: int = stream.content.headers.get("Content-Length") + if content_length: + description.length = int(content_length) + else: + # TODO: check statement validity + description.length = stream.content.headers.content_length + + return description + + @staticmethod + def serialize(item: Serializable, stream: List[int], length: List[int]): + encoded_json = item.to_json().encode() + stream.clear() + stream.extend(list(encoded_json)) + + length.clear() + length.append(len(stream)) + + async def _send(self): + # determine if we know the length we can send and whether we can tell if this is the end + is_length_known = self._is_end + + header = Header(type=self.type, id=self.identifier, end=self._is_end) + + header.payload_length = 0 + + if self._stream_length is not None: + # determine how many bytes we can send and if we are at the end + header.payload_length = min( + self._stream_length - self._send_offset, + TransportConstants.MAX_PAYLOAD_LENGTH, + ) + header.end = ( + self._send_offset + header.payload_length >= self._stream_length + ) + is_length_known = True + + self.sender.send_payload(header, self._stream, is_length_known, self._on_sent) + + async def _on_send(self, header: Header): + self._send_offset += header.payload_length + self._is_end = header.end + + if self._is_end: + self._task_completion_source.set_result(True) + else: + await self._send() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py new file mode 100644 index 000000000..a364d675a --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import List + +from botbuilder.streaming import StreamingRequest +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads.models import PayloadTypes, RequestPayload + +from .payload_disassembler import PayloadDisassembler + + +class RequestDisassembler(PayloadDisassembler): + def __init__( + self, sender: PayloadSender, identifier: UUID, request: StreamingRequest + ): + super().__init__(sender, identifier) + + self.request = request + + @property + def type(self) -> str: + return PayloadTypes.REQUEST + + async def get_stream(self) -> List[int]: + payload = RequestPayload(verb=self.request.verb, path=self.request.path) + + if self.request.streams: + payload.streams = [ + self.get_stream_description(content_stream) + for content_stream in self.request.streams + ] + + memory_stream: List[int] = [] + stream_length: List[int] = [] + # TODO: high probability stream length is not necessary + self.serialize(payload, memory_stream, stream_length) + + return memory_stream diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py index f582bd0f5..1c22fcf49 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py @@ -4,5 +4,14 @@ from .header import Header from .payload_types import PayloadTypes +from .request_payload import RequestPayload +from .serializable import Serializable +from .stream_description import StreamDescription -__all__ = ["Header", "PayloadTypes"] +__all__ = [ + "Header", + "PayloadTypes", + "RequestPayload", + "Serializable", + "StreamDescription", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py new file mode 100644 index 000000000..42d5241a4 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from typing import List + +from .serializable import Serializable +from .stream_description import StreamDescription + + +class RequestPayload(Serializable): + def __init__( + self, + *, + verb: str = None, + path: str = None, + streams: List[StreamDescription] = None + ): + self.verb = verb + self.path = path + self.streams = streams + + def to_json(self) -> str: + obj = {"verb": self.verb, "path": self.path} + + if self.streams: + obj["streams"] = [stream.to_dict() for stream in self.streams] + + return json.dumps(obj) + + def from_json(self, json_str: str): + obj = json.loads(json_str) + + self.verb = obj.get("verb") + self.path = obj.get("path") + stream_list = obj.get("streams") + + if stream_list: + self.streams = [ + StreamDescription().from_dict(stream_dict) + for stream_dict in stream_list + ] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py new file mode 100644 index 000000000..8c01830be --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from abc import ABC + + +# TODO: debate if this class is pertinent or should use msrest infrastructure +class Serializable(ABC): + def to_json(self) -> str: + raise NotImplementedError() + + def from_json(self, json_str: str): + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py new file mode 100644 index 000000000..0271234e5 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json + +from .serializable import Serializable + + +class StreamDescription(Serializable): + def __init__(self, *, id: str = None, content_type: str = None, length: int = None): + self.id = id + self.content_type = content_type + self.length = length + + def to_dict(self) -> dict: + obj = {"id": self.id, "contentType": self.content_type} + + if self.length is not None: + obj["length"] = self.length + + return obj + + def from_dict(self, json_dict: dict): + self.id = json_dict.get("id") + self.content_type = json_dict.get("contentType") + self.length = json_dict.get("length") + + def to_json(self) -> str: + return json.dumps(self.to_dict) + + def from_json(self, json_str: str): + obj = json.loads(json_str) + self.from_dict(obj) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py new file mode 100644 index 000000000..05689d8e5 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID + +from botbuilder.streaming import StreamingRequest +from botbuilder.streaming.payload_transport import PayloadSender + + +class SendOperations: + def __init__(self, payload_sender: PayloadSender): + self._payload_sender = payload_sender + + async def send_request(self, identifier: UUID, request: StreamingRequest): + disassembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py new file mode 100644 index 000000000..007f9b780 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from botbuilder.streaming.payloads import RequestManager +from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver + +from .request_handler import RequestHandler + + +class ProtocolAdapter: + def __init__( + self, + request_handler: RequestHandler, + request_manager: RequestManager, + payload_sender: PayloadSender, + payload_receiver: PayloadReceiver, + ): + self._request_handler = request_handler + self._request_manager = request_manager + self._payload_sender = payload_sender + self._payload_receiver = payload_receiver + + self._send_operations From 7224a5f3b44f5eb2270e4e4293b32d870ad89bd3 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 9 Apr 2020 00:03:25 -0700 Subject: [PATCH 06/37] POC almost ready for testing, changes on BFAdapter pending --- .../botbuilder/core/streaming/__init__.py | 3 +- .../streaming/streaming_request_handler.py | 167 +++++++++++++++++- .../botbuilder/core/streaming/version_info.py | 22 +++ .../botbuilder/streaming/__init__.py | 2 + .../payload_transport/payload_receiver.py | 8 + .../botbuilder/streaming/payloads/__init__.py | 14 +- .../streaming/payloads/assemblers/__init__.py | 14 ++ .../payloads/assemblers/assembler.py | 30 ++++ .../assemblers/payload_stream_assembler.py | 50 ++++++ .../assemblers/receive_request_assembler.py | 86 +++++++++ .../assemblers/receive_response_assembler.py | 84 +++++++++ .../streaming/payloads/content_stream.py | 23 +++ .../payloads/disassemblers/__init__.py | 12 +- .../disassemblers/cancel_disassembler.py | 22 +++ .../disassemblers/response_disassembler.py | 40 +++++ .../response_message_stream_disassembler.py | 27 +++ .../streaming/payloads/models/__init__.py | 2 + .../payloads/models/request_payload.py | 4 +- .../payloads/models/response_payload.py | 38 ++++ .../payloads/models/stream_description.py | 8 +- .../payloads/payload_assembler_manager.py | 71 ++++++++ .../streaming/payloads/request_manager.py | 6 +- .../streaming/payloads/send_operations.py | 57 +++++- .../streaming/payloads/stream_manager.py | 49 +++++ .../botbuilder/streaming/protocol_adapter.py | 63 ++++++- .../botbuilder/streaming/receive_request.py | 19 +- .../botbuilder/streaming/receive_response.py | 4 +- .../streaming/streaming_response.py | 17 +- .../transport/web_socket/web_socket_server.py | 62 ++++++- .../web_socket/web_socket_transport.py | 4 +- 30 files changed, 983 insertions(+), 25 deletions(-) create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/version_info.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py index efc2a34bd..68497b79b 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py @@ -3,5 +3,6 @@ from .bot_framework_http_adapter_base import BotFrameworkHttpAdapterBase from .streaming_activity_processor import StreamingActivityProcessor +from .version_info import VersionInfo -__all__ = ["BotFrameworkHttpAdapterBase", "StreamingActivityProcessor"] +__all__ = ["BotFrameworkHttpAdapterBase", "StreamingActivityProcessor", "VersionInfo"] diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index a1dd667a6..cf9457bf6 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -2,15 +2,26 @@ # Licensed under the MIT License. import platform +from collections import Counter +from http import HTTPStatus from datetime import datetime from logging import Logger -from typing import Dict +from typing import Dict, List from botbuilder.core import Bot -from botbuilder.streaming import RequestHandler, __title__, __version__ -from botbuilder.streaming.transport.web_socket import WebSocket +from botbuilder.schema import Activity, Attachment, ResourceResponse +from botbuilder.streaming import ( + RequestHandler, + ReceiveRequest, + StreamingRequest, + StreamingResponse, + __title__, + __version__, +) +from botbuilder.streaming.transport.web_socket import WebSocket, WebSocketServer from .streaming_activity_processor import StreamingActivityProcessor +from .version_info import VersionInfo class StreamingRequestHandler(RequestHandler): @@ -33,7 +44,115 @@ def __init__( self._logger = logger self._conversations: Dict[str, datetime] = {} self._user_agent = StreamingRequestHandler._get_user_agent() - self._server + self._server = WebSocketServer(web_socket, self) + self._server_is_connected = True + self._server.disconnected_event_handler = self._server_disconnected + self._service_url: str = None + + @property + def service_url(self) -> str: + return self._service_url + + async def listen(self): + await self._server.start() + # TODO: log it + + def has_conversation(self, conversation_id: str) -> bool: + return conversation_id in self._conversations + + def conversation_added_time(self, conversation_id: str) -> datetime: + added_time = self._conversations.get(conversation_id) + + if not added_time: + added_time = datetime.min + + return added_time + + def forget_conversation(self, conversation_id: str): + del self._conversations[conversation_id] + + async def process_request( + self, request: ReceiveRequest, logger: Logger, context: object + ) -> StreamingResponse: + response = StreamingResponse() + + # We accept all POSTs regardless of path, but anything else requires special treatment. + if not request.verb == StreamingRequest.POST: + return self._handle_custom_paths() + + # Convert the StreamingRequest into an activity the adapter can understand. + try: + body = request.read_body_as_str() + except Exception as error: + response.status_code = int(HTTPStatus.BAD_REQUEST) + # TODO: log error + + return response + + try: + # TODO: validate if should use deserialize or from_dict + activity: Activity = Activity.deserialize(body) + + # All activities received by this StreamingRequestHandler will originate from the same channel, but we won't + # know what that channel is until we've received the first request. + if not self.service_url: + self._service_url = activity.service_url + + # If this is the first time the handler has seen this conversation it needs to be added to the dictionary so + # the adapter is able to route requests to the correct handler. + if not self.has_conversation(activity.conversation.id): + self._conversations[activity.conversation.id] = datetime.now() + + """ + Any content sent as part of a StreamingRequest, including the request body + and inline attachments, appear as streams added to the same collection. The first + stream of any request will be the body, which is parsed and passed into this method + as the first argument, 'body'. Any additional streams are inline attachments that need + to be iterated over and added to the Activity as attachments to be sent to the Bot. + """ + + if len(request.streams) > 1: + stream_attachments = [ + Attachment(content_type=stream.content_type, content=stream.stream) + for stream in request.streams + ] + + if activity.attachments: + activity.attachments += stream_attachments + else: + activity.attachments = stream_attachments + + # Now that the request has been converted into an activity we can send it to the adapter. + adapter_response = await self._activity_processor.process_streaming_activity( + activity, self._bot.on_turn + ) + + # Now we convert the invokeResponse returned by the adapter into a StreamingResponse we can send back + # to the channel. + if not adapter_response: + response.status_code = int(HTTPStatus.OK) + else: + response.status_code = adapter_response.status + if adapter_response.body: + response.set_body(adapter_response.body) + + except Exception as error: + response.status_code = int(HTTPStatus.INTERNAL_SERVER_ERROR) + response.set_body(str(error)) + # TODO: log error + + return response + + async def send_activity(self, activity: Activity) -> ResourceResponse: + if activity.reply_to_id: + request_path = ( + f"/v3/conversations/{activity.conversation.id if activity.conversation else ''}/" + f"activities/{activity. reply_to_id}" + ) + else: + request_path = f"/v3/conversations/{activity.conversation.id if activity.conversation else ''}/activities" + + stream_attachments = self.updat @staticmethod def _get_user_agent() -> str: @@ -44,3 +163,43 @@ def _get_user_agent() -> str: platform_user_agent = f"({os_version}; {py_version})" user_agent = f"{package_user_agent} {platform_user_agent}" return user_agent + + async def update_attachment_streams(self, activity: Activity) -> List[object]: + if not activity or not activity.attachments: + return None + + def validate_int_list(obj: object) -> bool: + if not isinstance(obj, list): + return False + + return all(isinstance(element, int) for element in obj) + + stream_attachments = [ + attachment + for attachment in activity.attachments + if validate_int_list(attachment.content) + ] + + if stream_attachments: + activity.attachments = [ + attachment + for attachment in activity.attachments + if not validate_int_list(attachment.content) + ] + + def _handle_custom_paths( + self, request: ReceiveRequest, response: StreamingResponse + ) -> StreamingResponse: + if not request or not request.verb or not request.path: + response.status_code = int(HTTPStatus.BAD_REQUEST) + # TODO: log error + + return response + + if request.verb == StreamingRequest.GET and request.path == "/api/version": + response.status_code = int(HTTPStatus.OK) + response.set_body(VersionInfo(user_agent=self._user_agent)) + + return response + + return None diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py b/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py new file mode 100644 index 000000000..f866b8b7d --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json + +from botbuilder.streaming.payloads.models import Serializable + + +class VersionInfo(Serializable): + def __init__(self, *, user_agent: str = None): + self.user_agent = user_agent + + def to_json(self) -> str: + obj = {"userAgent": self.user_agent} + + return json.dumps(obj) + + def from_json(self, json_str: str) -> "ResponsePayload": + obj = json.loads(json_str) + + self.user_agent = obj.get("userAgent") + return self diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index 47fcd2269..f8449b357 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -6,6 +6,7 @@ # -------------------------------------------------------------------------- from .about import __version__, __title__ +from .protocol_adapter import ProtocolAdapter from .receive_request import ReceiveRequest from .receive_response import ReceiveResponse from .request_handler import RequestHandler @@ -13,6 +14,7 @@ from .streaming_response import StreamingResponse __all__ = [ + "ProtocolAdapter", "ReceiveRequest", "ReceiveResponse", "RequestHandler", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index 27c82f128..c7b72604f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -43,6 +43,14 @@ def connect(self, receiver: TransportReceiverBase): def _run_receive(self): asyncio.create_task(self._receive_packets()) + def subscribe( + self, + get_stream: Callable[[Header], List[int]], + receive_action: Callable[[Header, List[int]], int], + ): + self._get_stream = get_stream + self._receive_action = receive_action + def disconnect(self, event_args: DisconnectedEventArgs = None): did_disconnect = False diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py index 2bcb92561..06fd3ad21 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py @@ -1,8 +1,20 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from .content_stream import ContentStream from .header_serializer import HeaderSerializer +from .payload_assembler_manager import PayloadAssemblerManager from .request_manager import RequestManager from .response_message_stream import ResponseMessageStream +from .send_operations import SendOperations +from .stream_manager import StreamManager -__all__ = ["RequestManager", "ResponseMessageStream", "HeaderSerializer"] +__all__ = [ + "ContentStream", + "PayloadAssemblerManager", + "RequestManager", + "ResponseMessageStream", + "HeaderSerializer", + "SendOperations", + "StreamManager", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py new file mode 100644 index 000000000..0373292c4 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .assembler import Assembler +from .payload_stream_assembler import PayloadStreamAssembler +from .receive_request_assembler import ReceiveRequestAssembler +from .receive_response_assembler import ReceiveResponseAssembler + +__all__ = [ + "Assembler", + "PayloadStreamAssembler", + "ReceiveRequestAssembler", + "ReceiveResponseAssembler", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py new file mode 100644 index 000000000..3a0c048d2 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +from abc import ABC +from uuid import UUID + +from typing import List + +from botbuilder.streaming.payloads.models import Header + + +class Assembler(ABC): + def __init__(self, end: bool, identifier: UUID): + self.end = end + self.identifier = identifier + + def close(self): + raise NotImplementedError() + + def create_stream_from_payload(self) -> List[int]: + raise NotImplementedError() + + def get_payload_as_stream(self) -> List[int]: + raise NotImplementedError() + + def on_receive( + self, header: Header, stream: List[int], content_length: int + ) -> List[int]: + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py new file mode 100644 index 000000000..a452c19f4 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import List +from threading import Lock + +from botbuilder.streaming.payloads.models import Header + +from .assembler import Assembler + + +class PayloadStreamAssembler(Assembler): + def __init__( + self, + stream_manager: "StreamManager", + identifier: UUID, + type: str = None, + length: int = None, + ): + from botbuilder.streaming.payloads import StreamManager + + self._stream_manager = stream_manager or StreamManager() + self._stream: List[int] = [] + self._lock = Lock() + self.identifier = identifier + self.content_type = type + self.content_length = length + self.end: bool = None + + # TODO: highly probable this can be removed + def create_stream_from_payload(self) -> List[int]: + return [] + + # TODO: somewhat probable this can be removed + def get_payload_as_stream(self) -> List[int]: + with self._lock: + if self._stream is None: + self._stream = self.create_stream_from_payload() + + return self._stream + + def on_receive( + self, header: Header, stream: List[int], content_length: int + ) -> List[int]: + if header.end: + self.end = True + + def close(self): + self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py new file mode 100644 index 000000000..43631a8b7 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from uuid import UUID +from threading import Lock +from typing import Awaitable, Callable, List + +from botbuilder.streaming import ReceiveRequest +from botbuilder.streaming.payloads import ContentStream, StreamManager +from botbuilder.streaming.payloads.models import Header, RequestPayload + +from .assembler import Assembler + + +class ReceiveRequestAssembler(Assembler): + def __init__( + self, + header: Header, + stream_manager: StreamManager, + on_completed: Callable[[UUID, ReceiveRequest], Awaitable], + ): + if not header: + raise TypeError( + f"'header: {header.__class__.__name__}' argument can't be None" + ) + if not on_completed: + raise TypeError(f"'on_completed' argument can't be None") + + self._stream_manager = stream_manager + self._on_completed = on_completed + self.identifier = header.id + self._length = header.payload_length if header.end else None + self._lock = Lock() + self._stream: List[int] = None + + def create_stream_from_payload(self) -> List[int]: + return [None] * (self._length or 0) + + def get_payload_as_stream(self) -> List[int]: + with self._lock: + if self._stream is None: + self._stream = self.create_stream_from_payload() + + return self._stream + + def on_receive(self, header: Header, stream: List[int], content_length: int): + if header.end: + self.end = True + + # Execute the request on a separate Task + asyncio.create_task(self.process_request(stream)) + + def close(self): + self._stream_manager.close_stream(self.identifier) + + async def process_request(self, stream: List[int]): + request_payload = RequestPayload().from_json(bytes(stream).decode("utf8")) + + request = ReceiveRequest( + verb=request_payload.verb, path=request_payload.path, streams=[] + ) + + if request_payload.streams: + for stream_description in request_payload.streams: + try: + identifier = UUID(int=int(stream_description.id)) + except Exception: + raise ValueError( + f"Stream description id '{stream_description.id}' is not a Guid" + ) + + stream_assembler = self._stream_manager.get_payload_assembler( + identifier + ) + stream_assembler.content_type = stream_description.content_type + stream_assembler.content_length = stream_description.length + + content_stream = ContentStream( + identifier=identifier, assembler=stream_assembler + ) + content_stream.length = stream_description.length + content_stream.content_type = stream_description.content_type + request.streams.append(content_stream) + + await self._on_completed(self.identifier, request) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py new file mode 100644 index 000000000..123fadbc4 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from uuid import UUID +from threading import Lock +from typing import Awaitable, Callable, List + +from botbuilder.streaming import ReceiveResponse +from botbuilder.streaming.payloads import ContentStream, StreamManager +from botbuilder.streaming.payloads.models import Header, ResponsePayload + +from .assembler import Assembler + + +class ReceiveResponseAssembler(Assembler): + def __init__( + self, + header: Header, + stream_manager: StreamManager, + on_completed: Callable[[UUID, ReceiveResponse], Awaitable], + ): + if not header: + raise TypeError( + f"'header: {header.__class__.__name__}' argument can't be None" + ) + if not on_completed: + raise TypeError(f"'on_completed' argument can't be None") + + self._stream_manager = stream_manager + self._on_completed = on_completed + self.identifier = header.id + self._length = header.payload_length if header.end else None + self._lock = Lock() + self._stream: List[int] = None + + def create_stream_from_payload(self) -> List[int]: + return [None] * (self._length or 0) + + def get_payload_as_stream(self) -> List[int]: + with self._lock: + if self._stream is None: + self._stream = self.create_stream_from_payload() + + return self._stream + + def on_receive(self, header: Header, stream: List[int], content_length: int): + if header.end: + self.end = header.end + + # Execute the response on a separate Task + asyncio.create_task(self.process_response(stream)) + + def close(self): + self._stream_manager.close_stream(self.identifier) + + async def process_response(self, stream: List[int]): + response_payload = ResponsePayload().from_json(bytes(stream).decode("utf8")) + + response = ReceiveResponse(status_code=response_payload.status_code, streams=[]) + + if response_payload.streams: + for stream_description in response_payload.streams: + try: + identifier = UUID(int=int(stream_description.id)) + except Exception: + raise ValueError( + f"Stream description id '{stream_description.id}' is not a Guid" + ) + + stream_assembler = self._stream_manager.get_payload_assembler( + identifier + ) + stream_assembler.content_type = stream_description.content_type + stream_assembler.content_length = stream_description.length + + content_stream = ContentStream( + identifier=identifier, assembler=stream_assembler + ) + content_stream.length = stream_description.length + content_stream.content_type = stream_description.content_type + response.streams.append(content_stream) + + await self._on_completed(self.identifier, response) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py new file mode 100644 index 000000000..f61b44764 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID + +from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler + + +class ContentStream: + def __init__(self, identifier: UUID, assembler: PayloadStreamAssembler): + if not assembler: + raise TypeError( + f"'assembler: {assembler.__class__.__name__}' argument can't be None" + ) + + self.identifier = identifier + self._assembler = assembler + self.stream = self._assembler.get_payload_as_stream() + self.content_type: str = None + self.length: int = None + + def cancel(self): + self._assembler.close() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py index 2fe75682d..bc4270be5 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py @@ -1,8 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +from .cancel_disassembler import CancelDisassembler from .payload_disassembler import PayloadDisassembler from .request_disassembler import RequestDisassembler +from .response_disassembler import ResponseDisassembler +from .response_message_stream_disassembler import ResponseMessageStreamDisassembler -__all__ = ["PayloadDisassembler", "RequestDisassembler"] +__all__ = [ + "CancelDisassembler", + "PayloadDisassembler", + "RequestDisassembler", + "ResponseDisassembler", + "ResponseMessageStreamDisassembler", +] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py new file mode 100644 index 000000000..f1c2f49eb --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID + +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads.models import Header + + +class CancelDisassembler: + def __init__(self, *, sender: PayloadSender, identifier: UUID, type: str): + self._sender = sender + self._identifier = identifier + self._type = type + + async def disassemble(self): + header = Header(type=self._type, id=self._identifier, end=True) + + header.payload_length = 0 + + self._sender.send_payload(header, None, True, None) + return diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py new file mode 100644 index 000000000..48c083996 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import List + +from botbuilder.streaming import StreamingResponse +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads.models import PayloadTypes, ResponsePayload + +from .payload_disassembler import PayloadDisassembler + + +class ResponseDisassembler(PayloadDisassembler): + def __init__( + self, sender: PayloadSender, identifier: UUID, response: StreamingResponse + ): + super().__init__(sender, identifier) + + self.response = response + + @property + def type(self) -> str: + return PayloadTypes.RESPONSE + + async def get_stream(self) -> List[int]: + payload = ResponsePayload(status_code=self.response.status_code) + + if self.response.streams: + payload.streams = [ + self.get_stream_description(content_stream) + for content_stream in self.response.streams + ] + + memory_stream: List[int] = [] + stream_length: List[int] = [] + # TODO: high probability stream length is not necessary + self.serialize(payload, memory_stream, stream_length) + + return memory_stream diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py new file mode 100644 index 000000000..a7123c46e --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads import ResponseMessageStream +from botbuilder.streaming.payloads.models import PayloadTypes + +from .payload_disassembler import PayloadDisassembler + + +class ResponseMessageStreamDisassembler(PayloadDisassembler): + def __init__(self, sender: PayloadSender, content_stream: ResponseMessageStream): + super().__init__(sender, content_stream.id) + + self.content_stream = content_stream + + @property + def type(self) -> str: + return PayloadTypes.REQUEST + + async def get_stream(self) -> List[int]: + # TODO: align logic below to the shape of content_stream.content + stream: List[int] = list(str(self.content_stream.content).encode()) + + return stream diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py index 1c22fcf49..f0d3e2024 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py @@ -5,6 +5,7 @@ from .header import Header from .payload_types import PayloadTypes from .request_payload import RequestPayload +from .response_payload import ResponsePayload from .serializable import Serializable from .stream_description import StreamDescription @@ -12,6 +13,7 @@ "Header", "PayloadTypes", "RequestPayload", + "ResponsePayload", "Serializable", "StreamDescription", ] diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py index 42d5241a4..1003c292f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py @@ -28,7 +28,7 @@ def to_json(self) -> str: return json.dumps(obj) - def from_json(self, json_str: str): + def from_json(self, json_str: str) -> "RequestPayload": obj = json.loads(json_str) self.verb = obj.get("verb") @@ -40,3 +40,5 @@ def from_json(self, json_str: str): StreamDescription().from_dict(stream_dict) for stream_dict in stream_list ] + + return self diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py new file mode 100644 index 000000000..f1f41142c --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from typing import List + +from .serializable import Serializable +from .stream_description import StreamDescription + + +class ResponsePayload(Serializable): + def __init__( + self, *, status_code: int = None, streams: List[StreamDescription] = None + ): + self.status_code = status_code + self.streams = streams + + def to_json(self) -> str: + obj = {"statusCode": self.status_code} + + if self.streams: + obj["streams"] = [stream.to_dict() for stream in self.streams] + + return json.dumps(obj) + + def from_json(self, json_str: str) -> "ResponsePayload": + obj = json.loads(json_str) + + self.status_code = obj.get("statusCode") + stream_list = obj.get("streams") + + if stream_list: + self.streams = [ + StreamDescription().from_dict(stream_dict) + for stream_dict in stream_list + ] + + return self diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py index 0271234e5..395ef4b47 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py @@ -20,14 +20,16 @@ def to_dict(self) -> dict: return obj - def from_dict(self, json_dict: dict): + def from_dict(self, json_dict: dict) -> "StreamDescription": self.id = json_dict.get("id") self.content_type = json_dict.get("contentType") self.length = json_dict.get("length") + return self + def to_json(self) -> str: return json.dumps(self.to_dict) - def from_json(self, json_str: str): + def from_json(self, json_str: str) -> "StreamDescription": obj = json.loads(json_str) - self.from_dict(obj) + return self.from_dict(obj) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py new file mode 100644 index 000000000..9a02de7f8 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import Awaitable, Callable, Dict, List + +from botbuilder.streaming import ReceiveResponse, ReceiveRequest +from botbuilder.streaming.payloads.assemblers import ( + Assembler, + ReceiveRequestAssembler, + ReceiveResponseAssembler, +) +from botbuilder.streaming.payloads.models import Header, PayloadTypes + +from .stream_manager import StreamManager + + +class PayloadAssemblerManager: + def __init__( + self, + stream_manager: StreamManager, + on_receive_request: Callable[[UUID, ReceiveRequest], Awaitable], + on_receive_response: Callable[[UUID, ReceiveResponse], Awaitable], + ): + self._on_receive_request = on_receive_request + self._on_receive_response = on_receive_response + self._stream_manager = stream_manager + self._active_assemblers: Dict[UUID, Assembler] = {} + + def get_payload_stream(self, header: Header) -> List[int]: + if self._is_stream_payload(header): + return self._stream_manager.get_payload_stream(header) + elif not self._active_assemblers.get(header.id): + # a new requestId has come in, start a new task to process it as it is received + assembler = self._create_payload_assembler(header) + if assembler: + self._active_assemblers[header.id] = assembler + return assembler.get_payload_as_stream() + + return None + + def on_receive( + self, header: Header, content_stream: List[int], content_length: int + ): + if self._is_stream_payload(header): + self._stream_manager.on_receive(header, content_stream, content_length) + else: + assembler = self._active_assemblers.get(header.id) + if assembler: + assembler.on_receive(header, content_stream, content_length) + + # remove them when we are done + if header.end: + del self._active_assemblers[header.id] + + # ignore unknown header ids + + def _create_payload_assembler(self, header: Header) -> Assembler: + if header.type == PayloadTypes.REQUEST: + return ReceiveRequestAssembler( + header, self._stream_manager, self._on_receive_request + ) + elif header.type == PayloadTypes.RESPONSE: + return ReceiveResponseAssembler( + header, self._stream_manager, self._on_receive_response + ) + + return None + + def _is_stream_payload(self, header: Header) -> bool: + return header.type == PayloadTypes diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py index 2f3064969..07d4e0ccb 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py @@ -12,15 +12,15 @@ class RequestManager: def __init__(self, *, pending_requests: Dict[UUID, "Future[ReceiveResponse]"]): self._pending_requests = pending_requests or {} - def signal_response( - self, request_id: UUID, response: "Future[ReceiveResponse]" + async def signal_response( + self, request_id: UUID, response: ReceiveResponse ) -> bool: # TODO: dive more into this logic signal: Future = self._pending_requests.get(request_id) if signal: signal.set_result(response) # TODO: double check this - del self._pending_requests[request_id] + # del self._pending_requests[request_id] return True diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py index 05689d8e5..1b94f6e08 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py @@ -1,10 +1,18 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import asyncio from uuid import UUID -from botbuilder.streaming import StreamingRequest +from botbuilder.streaming import StreamingRequest, StreamingResponse from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads.disassemblers import ( + CancelDisassembler, + RequestDisassembler, + ResponseDisassembler, + ResponseMessageStreamDisassembler, +) +from botbuilder.streaming.payloads.models import PayloadTypes class SendOperations: @@ -12,4 +20,49 @@ def __init__(self, payload_sender: PayloadSender): self._payload_sender = payload_sender async def send_request(self, identifier: UUID, request: StreamingRequest): - disassembler + disassembler = RequestDisassembler(self._payload_sender, identifier, request) + + await disassembler.disassemble() + + if request.streams: + tasks = [ + ResponseMessageStreamDisassembler( + self._payload_sender, content_stream + ).disassemble() + for content_stream in request.streams + ] + + await asyncio.gather(*tasks) + + async def send_response(self, identifier: UUID, response: StreamingResponse): + disassembler = ResponseDisassembler(self._payload_sender, identifier, response) + + await disassembler.disassemble() + + if response.streams: + tasks = [ + ResponseMessageStreamDisassembler( + self._payload_sender, content_stream + ).disassemble() + for content_stream in response.streams + ] + + await asyncio.gather(*tasks) + + async def send_cancel_all(self, identifier: UUID): + disassembler = CancelDisassembler( + sender=self._payload_sender, + identifier=identifier, + type=PayloadTypes.CANCEL_ALL, + ) + + await disassembler.disassemble() + + async def send_cancel_stream(self, identifier: UUID): + disassembler = CancelDisassembler( + sender=self._payload_sender, + identifier=identifier, + type=PayloadTypes.CANCEL_STREAM, + ) + + await disassembler.disassemble() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py new file mode 100644 index 000000000..e58b5ee13 --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from uuid import UUID +from typing import Callable, Dict, List + +from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler +from botbuilder.streaming.payloads.models import Header + + +class StreamManager: + def __init__( + self, on_cancel_stream: Callable[[PayloadStreamAssembler], None] = None + ): + self._on_cancel_stream = on_cancel_stream or (lambda ocs: None) + self._active_assemblers: Dict[UUID, PayloadStreamAssembler] = {} + + def get_payload_assembler(self, identifier: UUID) -> PayloadStreamAssembler: + self._active_assemblers[identifier] = self._active_assemblers.get( + identifier, PayloadStreamAssembler(self, identifier) + ) + + return self._active_assemblers[identifier] + + def get_payload_stream(self, header: Header) -> List[int]: + assembler = self.get_payload_assembler(header.id) + + return assembler.get_payload_as_stream() + + def on_receive( + self, header: Header, content_stream: List[int], content_length: int + ): + assembler = self._active_assemblers.get(header.id) + + if assembler: + assembler.on_receive(header, content_stream, content_length) + + def close_stream(self, identifier: UUID): + assembler = self._active_assemblers.get(identifier) + + if assembler: + del self._active_assemblers[identifier] + stream = assembler.get_payload_as_stream() + if ( + assembler.content_length + and len(stream) < assembler.content_length + or not assembler.end + ): + self._on_cancel_stream(assembler) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py index 007f9b780..b1aeabeb6 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py @@ -1,10 +1,21 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import asyncio +from uuid import UUID, uuid4 -from botbuilder.streaming.payloads import RequestManager +from botbuilder.streaming.payloads import ( + PayloadAssemblerManager, + RequestManager, + SendOperations, + StreamManager, +) +from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver +from .receive_request import ReceiveRequest +from .receive_response import ReceiveResponse from .request_handler import RequestHandler +from .streaming_request import StreamingRequest class ProtocolAdapter: @@ -14,10 +25,58 @@ def __init__( request_manager: RequestManager, payload_sender: PayloadSender, payload_receiver: PayloadReceiver, + handler_context: object = None, ): self._request_handler = request_handler self._request_manager = request_manager self._payload_sender = payload_sender self._payload_receiver = payload_receiver + self._handler_context = handler_context - self._send_operations + self._send_operations = SendOperations(self._payload_sender) + # TODO: might be able to remove + self._stream_manager = StreamManager(self._on_cancel_stream) + self._assembler_manager = PayloadAssemblerManager( + self._stream_manager, self._on_receive_request, self._on_receive_response + ) + + self._payload_receiver.subscribe( + self._assembler_manager.get_payload_stream, + self._assembler_manager.on_receive, + ) + + async def send_request(self, request: StreamingRequest) -> ReceiveResponse: + if not request: + raise TypeError( + f"'request: {request.__class__.__name__}' argument can't be None" + ) + + request_id = uuid4() + response_task = self._request_manager.get_response(request_id) + request_task = self._send_operations.send_request(request_id, request) + + [request, _] = await asyncio.gather(request_task, response_task) + + return request + + async def _on_receive_request(self, identifier: UUID, request: ReceiveRequest): + # request is done, we can handle it + if self._request_handler: + response = await self._request_handler.process_request( + request, None, self._handler_context + ) + + if response: + await self._send_operations.send_response(identifier, response) + + async def _on_receive_response(self, identifier: UUID, response: ReceiveResponse): + # we received the response to something, signal it + await self._request_manager.signal_response(identifier, response) + + def _on_cancel_stream(self, content_stream_assembler: PayloadStreamAssembler): + # TODO: on original C# code content_stream_assembler is typed as IAssembler + asyncio.create_task( + self._send_operations.send_cancel_stream( + content_stream_assembler.identifier + ) + ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py index da0738698..73cc38ac2 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py @@ -3,9 +3,24 @@ from typing import List +from botbuilder.streaming.payloads import ContentStream + class ReceiveRequest: - def __init__(self, *, verb: str = None, path: str = None, streams: List[object]): + def __init__( + self, *, verb: str = None, path: str = None, streams: List[ContentStream] + ): self.verb = verb self.path = path - self.streams = streams or [] + self.streams: List[ContentStream] = streams or [] + + def read_body_as_str(self) -> str: + try: + content_stream = self.streams[0] if self.streams else None + + if not content_stream: + return "" + + return bytes(content_stream.stream).decode("utf8") + except Exception as error: + raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index e1831cfa0..401cae383 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -3,8 +3,10 @@ from typing import List +from botbuilder.streaming.payloads import ContentStream + class ReceiveResponse: - def __init__(self, status_code: int = None, streams: List[object] = None): + def __init__(self, status_code: int = None, streams: List[ContentStream] = None): self.status_code = status_code self.streams = streams diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py index 8de0fb31b..566d89563 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py @@ -1,9 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import List +import json +from typing import List, Union +from msrest.serialization import Model from botbuilder.streaming.payloads import ResponseMessageStream +from botbuilder.streaming.payloads.models import Serializable class StreamingResponse: @@ -22,6 +25,18 @@ def add_stream(self, content: object): self.streams.append(ResponseMessageStream(content=content)) + def set_body(self, body: Union[str, Serializable, Model]): + # TODO: verify if msrest.serialization.Model is necessary + if not body: + return + + if isinstance(body, Serializable): + body = body.to_json() + elif isinstance(body, Model): + body = json.dumps(body.as_dict()) + + self.add_stream(list(body.encode())) + @staticmethod def create_response(status_code: int, body: object) -> "StreamingResponse": response = StreamingResponse(status_code=status_code) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index 37bbff37b..bd26321ae 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -1,9 +1,18 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from botbuilder.streaming import RequestHandler +from asyncio import Future +from typing import Callable + +from botbuilder.streaming import ( + ProtocolAdapter, + ReceiveResponse, + RequestHandler, + StreamingRequest, +) from botbuilder.streaming.payloads import RequestManager from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver +from botbuilder.streaming.transport import DisconnectedEventArgs from .web_socket import WebSocket from .web_socket_transport import WebSocketTransport @@ -20,6 +29,10 @@ def __init__(self, socket: WebSocket, request_handler: RequestHandler): f"'request_handler: {request_handler.__class__.__name__}' argument can't be None" ) + self.disconnected_event_handler: Callable[ + [object, DisconnectedEventArgs], None + ] = None + self._web_socket_transport = WebSocketTransport(socket) self._request_handler = request_handler self._request_manager = RequestManager() @@ -27,3 +40,50 @@ def __init__(self, socket: WebSocket, request_handler: RequestHandler): self._sender.disconnected = self._on_connection_disconnected self._receiver = PayloadReceiver() self._receiver.disconnected = self._on_connection_disconnected + self._protocol_adapter = ProtocolAdapter( + self._request_handler, self._request_manager, self._sender, self._receiver + ) + self._closed_signal: Future = None + self._is_disconnecting: bool = False + + @property + def is_connected(self) -> bool: + return self._sender.is_connected and self._receiver.is_connected + + async def start(self): + self._closed_signal = Future() + self._sender.connect(self._web_socket_transport) + self._receiver.connect(self._web_socket_transport) + + return self._closed_signal + + async def send(self, request: StreamingRequest) -> ReceiveResponse: + if not request: + raise TypeError( + f"'request: {request.__class__.__name__}' argument can't be None" + ) + + if not self._sender.is_connected or not self._sender.is_connected: + raise RuntimeError("The server is not connected") + + return await self._protocol_adapter.send_request() + + def disconnect(self): + self._sender.disconnect() + self._receiver.disconnect() + + def _on_connection_disconnected(self, sender: object, event_args: object): + if not self._is_disconnecting: + self._is_disconnecting = True + + if self._closed_signal: + self._closed_signal.set_result("close") + self._closed_signal = None + + if sender in [self._sender, self._receiver]: + sender.disconnect() + + if self.disconnected_event_handler: + self.disconnected_event_handler(self, DisconnectedEventArgs.empty) + + self._is_disconnecting = False diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index c8bdd4696..373e6abba 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -1,13 +1,15 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from botbuilder.streaming.transport import TransportReceiverBase, TransportSenderBase + from .web_socket import WebSocket from .web_socket_message_type import WebSocketMessageType from .web_socket_close_status import WebSocketCloseStatus from .web_socket_state import WebSocketState -class WebSocketTransport: +class WebSocketTransport(TransportReceiverBase, TransportSenderBase): def __init__(self, web_socket: WebSocket): self._socket = web_socket From e23329728ae76aeb57d21a7e53bdf34c223d95eb Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Fri, 10 Apr 2020 16:05:35 -0700 Subject: [PATCH 07/37] POC waiting on client injection in connector --- .../bot_framework_http_adapter_base.py | 65 ++++++++++++++++-- .../core/streaming/streaming_http_client.py | 43 ++++++++++++ .../streaming/streaming_request_handler.py | 68 ++++++++++++++++++- .../botbuilder/streaming/receive_response.py | 29 +++++++- .../botbuilder/streaming/streaming_request.py | 17 ++++- .../streaming/streaming_response.py | 7 +- 6 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index 09ee9d9d6..8d87cb5d9 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -1,18 +1,71 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import List +from typing import Awaitable, Callable, List -from botbuilder.core import Bot, BotFrameworkAdapter, BotFrameworkAdapterSettings -from botframework.connector.auth import ClaimsIdentity +from botbuilder.core import ( + Bot, + BotFrameworkAdapter, + BotFrameworkAdapterSettings, + InvokeResponse, + TurnContext, +) +from botbuilder.schema import Activity +from botframework.connector.aio import ConnectorClient +from botframework.connector.auth import ( + ClaimsIdentity, + MicrosoftAppCredentials, + MicrosoftGovernmentAppCredentials, +) from .streaming_activity_processor import StreamingActivityProcessor +from .streaming_request_handler import StreamingRequestHandler class BotFrameworkHttpAdapterBase(BotFrameworkAdapter, StreamingActivityProcessor): def __init__(self, settings: BotFrameworkAdapterSettings): super().__init__(self, settings) - self._connected_bot: Bot = None - self._claims_identity: ClaimsIdentity = None - self._request_handlers: List[object] = None + self.connected_bot: Bot = None + self.claims_identity: ClaimsIdentity = None + self.request_handlers: List[StreamingRequestHandler] = None + + async def process_streaming_activity( + self, + activity: Activity, + bot_callback_handler: Callable[[TurnContext], Awaitable], + ) -> InvokeResponse: + if not activity: + raise TypeError( + f"'activity: {activity.__class__.__name__}' argument can't be None" + ) + + """ + If a conversation has moved from one connection to another for the same Channel or Skill and + hasn't been forgotten by the previous StreamingRequestHandler. The last requestHandler + the conversation has been associated with should always be the active connection. + """ + request_handler = [ + handler + for handler in self.request_handlers + if handler.service_url == activity.service_url + and handler.has_conversation(activity.conversation.id) + ] + request_handler = request_handler[-1] if request_handler else None + context = TurnContext(self, activity) + + if self.claims_identity: + context.turn_state[self.BOT_IDENTITY_KEY] = self.claims_identity + + connector_client = self.create_connector_client() + + def _create_streaming_connector_client( + self, activity: Activity, request_handler: StreamingRequestHandler + ) -> ConnectorClient: + empty_credentials = ( + MicrosoftAppCredentials.empty() + if self._channel_provider and self._channel_provider.is_government() + else MicrosoftGovernmentAppCredentials.empty() + ) + + streaming_client = diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py new file mode 100644 index 000000000..a4338cb2d --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from http import HTTPStatus +from logging import Logger + +from botbuilder.streaming import StreamingRequest + +from .streaming_request_handler import StreamingRequestHandler + + +class StreamingHttpClient: + def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = None): + if not request_handler: + raise TypeError(f"'request_handler: {request_handler.__class__.__name__}' argument can't be None") + self._request_handler = request_handler + self._logger = logger + + async def send(self, request: object) -> object: + # TODO: validate form of request to perform operations + streaming_request = StreamingRequest( + path=request.path[request.path.index("/v3"):], + verb=request.method + ) + streaming_request.set_body(request.content) + + return await self._send_request(streaming_request) + + async def _send_request(self, request: StreamingRequest) -> object: + try: + server_response = await self._request_handler.send_streaming_request(request) + + if not server_response: + raise Exception("Server response from streaming request is None") + + if server_response.status_code == HTTPStatus.OK: + # TODO: this should be an object read from json + return server_response.read_body_as_str() + except Exception as error: + # TODO: log error + pass + + return None diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index cf9457bf6..48af65c93 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import platform -from collections import Counter from http import HTTPStatus from datetime import datetime from logging import Logger @@ -13,17 +12,25 @@ from botbuilder.streaming import ( RequestHandler, ReceiveRequest, + ReceiveResponse, StreamingRequest, StreamingResponse, __title__, __version__, ) +from botbuilder.streaming.transport import DisconnectedEventArgs from botbuilder.streaming.transport.web_socket import WebSocket, WebSocketServer from .streaming_activity_processor import StreamingActivityProcessor from .version_info import VersionInfo +class StreamContent: + def __init__(self, stream: List[int], *, headers: Dict[str, str] = None): + self.stream = stream + self.headers: Dict[str, str] = headers if headers is not None else {} + + class StreamingRequestHandler(RequestHandler): def __init__( self, @@ -152,7 +159,48 @@ async def send_activity(self, activity: Activity) -> ResourceResponse: else: request_path = f"/v3/conversations/{activity.conversation.id if activity.conversation else ''}/activities" - stream_attachments = self.updat + stream_attachments = self._update_attachment_streams(activity) + request = StreamingRequest.create_post(request_path) + request.set_body(activity) + if stream_attachments: + for attachment in stream_attachments: + # TODO: might be necessary to serialize this before adding + request.add_stream(attachment) + + try: + if not self._server_is_connected: + raise Exception( + "Error while attempting to send: Streaming transport is disconnected." + ) + + server_response = await self._server.send(request) + + if server_response.status_code == HTTPStatus.OK: + return server_response.read_body_as_json(ResourceResponse) + except Exception: + # TODO: log error + pass + + return None + + async def send_streaming_request( + self, request: StreamingRequest + ) -> ReceiveResponse: + try: + if not self._server_is_connected: + raise Exception( + "Error while attempting to send: Streaming transport is disconnected." + ) + + server_response = await self._server.send(request) + + if server_response.status_code == HTTPStatus.OK: + return server_response.read_body_as_json(ReceiveResponse) + except Exception: + # TODO: log error + pass + + return None @staticmethod def _get_user_agent() -> str: @@ -164,7 +212,7 @@ def _get_user_agent() -> str: user_agent = f"{package_user_agent} {platform_user_agent}" return user_agent - async def update_attachment_streams(self, activity: Activity) -> List[object]: + def _update_attachment_streams(self, activity: Activity) -> List[object]: if not activity or not activity.attachments: return None @@ -187,6 +235,20 @@ def validate_int_list(obj: object) -> bool: if not validate_int_list(attachment.content) ] + # TODO: validate StreamContent parallel + return [ + StreamContent( + attachment.content, + headers={"Content-Type": attachment.content_type}, + ) + for attachment in stream_attachments + ] + + return None + + def _server_disconnected(self, sender: object, event: DisconnectedEventArgs): + self._server_is_connected = False + def _handle_custom_paths( self, request: ReceiveRequest, response: StreamingResponse ) -> StreamingResponse: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index 401cae383..3a9211a5b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -1,12 +1,39 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import List +from typing import List, Union, Type +from msrest.serialization import Model from botbuilder.streaming.payloads import ContentStream +from botbuilder.streaming.payloads.models import Serializable class ReceiveResponse: def __init__(self, status_code: int = None, streams: List[ContentStream] = None): self.status_code = status_code self.streams = streams + + def read_body_as_json( + self, cls: Type[Model, Serializable] + ) -> Union[Model, Serializable]: + try: + body_str = self.read_body_as_str() + + if issubclass(cls, Serializable): + body = cls().from_json(body_str) + elif isinstance(cls, Model): + body = cls.deserialize(body_str) + return body + except Exception as error: + raise error + + def read_body_as_str(self) -> str: + try: + content_stream = self.streams[0] if self.streams else None + + if not content_stream: + return "" + + return bytes(content_stream.stream).decode("utf8") + except Exception as error: + raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py index 1d3d5f319..a69923d0d 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -1,10 +1,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import json from uuid import UUID, uuid4 -from typing import List +from typing import List, Union +from msrest.serialization import Model from botbuilder.streaming.payloads import ResponseMessageStream +from botbuilder.streaming.payloads.models import Serializable class StreamingRequest: @@ -54,6 +57,18 @@ def create_put(path: str = None, body: object = None) -> "StreamingRequest": def create_delete(path: str = None, body: object = None) -> "StreamingRequest": return StreamingRequest.create_request("DELETE", path, body) + def set_body(self, body: Union[str, Serializable, Model]): + # TODO: verify if msrest.serialization.Model is necessary + if not body: + return + + if isinstance(body, Serializable): + body = body.to_json() + elif isinstance(body, Model): + body = json.dumps(body.as_dict()) + + self.add_stream(list(body.encode())) + def add_stream(self, content: object, stream_id: UUID = None): if not content: raise TypeError( diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py index 566d89563..f052af399 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import json +from uuid import UUID, uuid4 from typing import List, Union from msrest.serialization import Model @@ -16,14 +17,16 @@ def __init__( self.status_code = status_code self.streams = streams - def add_stream(self, content: object): + def add_stream(self, content: object, identifier: UUID = None): if not content: raise TypeError("content can't be None") if self.streams is None: self.streams: List[ResponseMessageStream] = [] - self.streams.append(ResponseMessageStream(content=content)) + self.streams.append( + ResponseMessageStream(id=identifier or uuid4(), content=content) + ) def set_body(self, body: Union[str, Serializable, Model]): # TODO: verify if msrest.serialization.Model is necessary From cb2ba6dc9c33a3c526c5c81ff6d68ad4aef369bf Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Fri, 10 Apr 2020 16:06:30 -0700 Subject: [PATCH 08/37] black: POC waiting on client injection in connector --- .../streaming/bot_framework_http_adapter_base.py | 2 +- .../core/streaming/streaming_http_client.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index 8d87cb5d9..dcc4220d3 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -68,4 +68,4 @@ def _create_streaming_connector_client( else MicrosoftGovernmentAppCredentials.empty() ) - streaming_client = + streaming_client diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index a4338cb2d..adbdd6044 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -12,15 +12,16 @@ class StreamingHttpClient: def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = None): if not request_handler: - raise TypeError(f"'request_handler: {request_handler.__class__.__name__}' argument can't be None") + raise TypeError( + f"'request_handler: {request_handler.__class__.__name__}' argument can't be None" + ) self._request_handler = request_handler self._logger = logger - + async def send(self, request: object) -> object: # TODO: validate form of request to perform operations streaming_request = StreamingRequest( - path=request.path[request.path.index("/v3"):], - verb=request.method + path=request.path[request.path.index("/v3") :], verb=request.method ) streaming_request.set_body(request.content) @@ -28,7 +29,9 @@ async def send(self, request: object) -> object: async def _send_request(self, request: StreamingRequest) -> object: try: - server_response = await self._request_handler.send_streaming_request(request) + server_response = await self._request_handler.send_streaming_request( + request + ) if not server_response: raise Exception("Server response from streaming request is None") From c079dc11513d4649941aa3f64f0fc85912c37525 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Mon, 20 Apr 2020 11:45:43 -0700 Subject: [PATCH 09/37] POC for http client injection in connector --- .../botframework/connector/__init__.py | 6 ++- .../connector/aio/_connector_client_async.py | 25 ++++++++++-- .../connector/aiohttp_bf_pipeline.py | 39 +++++++++++++++++++ .../bot_framework_sdk_client_async.py | 37 ++++++++++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py create mode 100644 libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py diff --git a/libraries/botframework-connector/botframework/connector/__init__.py b/libraries/botframework-connector/botframework/connector/__init__.py index 47e6ad952..5dd822c31 100644 --- a/libraries/botframework-connector/botframework/connector/__init__.py +++ b/libraries/botframework-connector/botframework/connector/__init__.py @@ -14,6 +14,10 @@ from .emulator_api_client import EmulatorApiClient from .version import VERSION -__all__ = ["Channels", "ConnectorClient", "EmulatorApiClient"] +# TODO: Experimental +from .aiohttp_bf_pipeline import AiohttpBfPipeline +from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration + +__all__ = ["AiohttpBfPipeline", "Channels", "ConnectorClient", "EmulatorApiClient"] __version__ = VERSION diff --git a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py index ff6b9b314..67b8ff2aa 100644 --- a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py +++ b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py @@ -9,7 +9,9 @@ # regenerated. # -------------------------------------------------------------------------- -from msrest.async_client import SDKClientAsync +from typing import Optional, Type + +from msrest.async_client import SDKClientAsync, AsyncPipeline from msrest import Serializer, Deserializer from .._configuration import ConnectorClientConfiguration @@ -18,7 +20,14 @@ from .. import models -class ConnectorClient(SDKClientAsync): +# TODO: experimental +from ..bot_framework_sdk_client_async import ( + BotFrameworkSDKClientAsync, + BotFrameworkConnectorConfiguration, +) + + +class ConnectorClient(BotFrameworkSDKClientAsync): """The Bot Connector REST API allows your bot to send and receive messages to channels configured in the [Bot Framework Developer Portal](https://dev.botframework.com). The Connector service uses industry-standard REST and JSON over HTTPS. @@ -49,9 +58,17 @@ class ConnectorClient(SDKClientAsync): :param str base_url: Service URL """ - def __init__(self, credentials, base_url=None): + def __init__( + self, + credentials, + base_url=None, + *, + pipeline_class: Optional[Type[AsyncPipeline]] = None + ): - self.config = ConnectorClientConfiguration(credentials, base_url) + self.config = BotFrameworkConnectorConfiguration( + credentials, base_url, pipeline=pipeline_class + ) super(ConnectorClient, self).__init__(self.config) client_models = { diff --git a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py new file mode 100644 index 000000000..151b4d7e4 --- /dev/null +++ b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from msrest.async_client import ServiceClientAsync +from msrest.pipeline import AsyncPipeline, AsyncHTTPPolicy, SansIOHTTPPolicy +from msrest.pipeline.aiohttp import AioHTTPSender +from msrest.universal_http.aiohttp import AioHTTPSender as Driver +from msrest.pipeline.async_requests import AsyncRequestsCredentialsPolicy +from msrest.pipeline.universal import RawDeserializer + +from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration + + +class AiohttpBfPipeline(AsyncPipeline): + def __init__(self, config: BotFrameworkConnectorConfiguration): + creds = config.credentials + + policies = [ + config.user_agent_policy, # UserAgent policy + RawDeserializer(), # Deserialize the raw bytes + config.http_logger_policy, # HTTP request/response log + ] # type: List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] + if creds: + if isinstance(creds, (AsyncHTTPPolicy, SansIOHTTPPolicy)): + policies.insert(1, creds) + else: + # Assume this is the old credentials class, and then requests. Wrap it. + policies.insert(1, AsyncRequestsCredentialsPolicy(creds)) + + super().__init__(policies, AioHTTPSender(BFAioHTTPDriver)) + + +class BFAioHTTPDriver(Driver): + """AioHttp HTTP sender implementation. + """ + + def __del__(self): + asyncio.create_task(self._session.close()) diff --git a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py new file mode 100644 index 000000000..7b92265d3 --- /dev/null +++ b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Optional, Type + +from msrest.async_client import SDKClientAsync, ServiceClientAsync +from msrest.pipeline import AsyncPipeline + + +from ._configuration import ConnectorClientConfiguration + + +class BotFrameworkConnectorConfiguration(ConnectorClientConfiguration): + def __init__( + self, + credentials, + base_url: str, + *, + pipeline: Optional[Type[AsyncPipeline]] = None + ): + super().__init__(credentials, base_url) + + if pipeline: + self.pipeline = pipeline(self) + + +class BotFrameworkSDKClientAsync(SDKClientAsync): + def __init__(self, config: BotFrameworkConnectorConfiguration) -> None: + super().__init__(config) + self._client = BotFrameworkServiceClientAsync(config) + + +class BotFrameworkServiceClientAsync(ServiceClientAsync): + def __init__(self, config: BotFrameworkConnectorConfiguration) -> None: + super(ServiceClientAsync, self).__init__(config) + + self.config.pipeline = config.pipeline or self._create_default_pipeline() From 0ae3fd96a46d8747592508c4238c5ee9c52e9573 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 23 Apr 2020 14:57:25 -0700 Subject: [PATCH 10/37] got rid of importing errors when loading libraries. Currently in the process of testing. --- .../botbuilder/core/bot_framework_adapter.py | 8 ++ ...tp_channel_service_exception_middleware.py | 12 +- .../botbuilder/core/streaming/__init__.py | 11 +- .../core/streaming/aiohttp_web_socket.py | 61 +++++++++ .../bot_framework_http_adapter_base.py | 109 +++++++++++++++- .../core/streaming/streaming_http_client.py | 59 ++++++++- .../aiohttp/bot_framework_http_adapter.py | 121 ++++++++++++++++-- .../botbuilder/streaming/__init__.py | 4 +- .../assemblers/payload_stream_assembler.py | 6 +- .../assemblers/receive_request_assembler.py | 12 +- .../assemblers/receive_response_assembler.py | 14 +- .../disassemblers/request_disassembler.py | 7 +- .../disassemblers/response_disassembler.py | 7 +- .../payloads/payload_assembler_manager.py | 6 +- .../streaming/payloads/request_manager.py | 14 +- .../streaming/payloads/send_operations.py | 10 +- .../botbuilder/streaming/receive_response.py | 15 ++- .../botbuilder/streaming/streaming_request.py | 17 ++- .../transport/web_socket/__init__.py | 3 +- .../transport/web_socket/web_socket.py | 34 ++++- .../web_socket/web_socket_transport.py | 10 +- .../botframework/connector/__init__.py | 4 +- .../connector/aio/_connector_client_async.py | 15 ++- .../connector/aiohttp_bf_pipeline.py | 7 +- .../bot_framework_sdk_client_async.py | 14 +- 25 files changed, 497 insertions(+), 83 deletions(-) create mode 100644 libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py diff --git a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py index 7edf9da2e..3a50b11c2 100644 --- a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py +++ b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py @@ -1219,6 +1219,14 @@ async def exchange_token_from_credentials( exchange_request.token, ) + def can_process_outgoing_activity(self, activity: Activity) -> bool: + return False + + async def process_outgoing_activity( + self, turn_context: TurnContext, activity: Activity + ) -> ResourceResponse: + raise NotImplementedError() + @staticmethod def key_for_connector_client(service_url: str, app_id: str, scope: str): return f"{service_url}:{app_id}:{scope}" diff --git a/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py b/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py index 7c5091121..fb9d9d3cd 100644 --- a/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py +++ b/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import traceback + from aiohttp.web import ( middleware, HTTPNotImplemented, @@ -25,5 +27,11 @@ async def aiohttp_error_middleware(request, handler): raise HTTPUnauthorized() except KeyError: raise HTTPNotFound() - except Exception: - raise HTTPInternalServerError() + except Exception as error: + try: + raise error + raise HTTPInternalServerError() + except: + pass + + traceback.print_exc() diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py index 68497b79b..bcb8890bb 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py @@ -1,8 +1,17 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# TODO: this class is gonna be moved eventually to integration +from .aiohttp_web_socket import AiohttpWebSocket from .bot_framework_http_adapter_base import BotFrameworkHttpAdapterBase from .streaming_activity_processor import StreamingActivityProcessor +from .streaming_request_handler import StreamingRequestHandler from .version_info import VersionInfo -__all__ = ["BotFrameworkHttpAdapterBase", "StreamingActivityProcessor", "VersionInfo"] +__all__ = [ + "AiohttpWebSocket", + "BotFrameworkHttpAdapterBase", + "StreamingActivityProcessor", + "StreamingRequestHandler", + "VersionInfo", +] diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py new file mode 100644 index 000000000..94e6e54a7 --- /dev/null +++ b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py @@ -0,0 +1,61 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from typing import Any, Optional, Union + +from aiohttp import ClientWebSocketResponse, WSMsgType, ClientSession +from aiohttp.web import WebSocketResponse + +from botbuilder.streaming.transport.web_socket import ( + WebSocket, + WebSocketMessage, + WebSocketCloseStatus, + WebSocketMessageType, + WebSocketState, +) + + +class AiohttpWebSocket(WebSocket): + def __init__( + self, + aiohttp_ws: Union[WebSocketResponse, ClientWebSocketResponse], + session: Optional[ClientSession] = None, + ): + self._aiohttp_ws = aiohttp_ws + self._session = session + + def dispose(self): + if self._session: + asyncio.create_task(self._session.close()) + + async def close(self, close_status: WebSocketCloseStatus, status_description: str): + await self._aiohttp_ws.close( + code=int(close_status), message=status_description.encode("utf8") + ) + + async def receive(self) -> WebSocketMessage: + message = await self._aiohttp_ws.receive() + + return WebSocketMessage( + message_type=WebSocketMessageType(int(message.type)), + data=list(str(message.data).encode("utf8")) + if message.type == WSMsgType.TEXT + else list(message.data), + ) + + async def send( + self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool + ): + if message_type == WebSocketMessageType.BINARY: + await self._aiohttp_ws.send_bytes(buffer) + elif message_type == WebSocketMessageType.TEXT: + await self._aiohttp_ws.send_str(buffer) + else: + raise RuntimeError( + f"AiohttpWebSocket - message_type: {message_type} currently not supported" + ) + + @property + async def status(self) -> WebSocketState: + return WebSocketState.CLOSED if self._aiohttp_ws.closed else WebSocketState.OPEN diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index dcc4220d3..31a5829df 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from http import HTTPStatus from typing import Awaitable, Callable, List +from aiohttp import ClientSession from botbuilder.core import ( Bot, BotFrameworkAdapter, @@ -10,7 +12,8 @@ InvokeResponse, TurnContext, ) -from botbuilder.schema import Activity +from botbuilder.schema import Activity, ActivityTypes, ResourceResponse +from botframework.connector import AsyncBfPipeline from botframework.connector.aio import ConnectorClient from botframework.connector.auth import ( ClaimsIdentity, @@ -20,11 +23,13 @@ from .streaming_activity_processor import StreamingActivityProcessor from .streaming_request_handler import StreamingRequestHandler +from .streaming_http_client import StreamingHttpDriver +from .aiohttp_web_socket import AiohttpWebSocket class BotFrameworkHttpAdapterBase(BotFrameworkAdapter, StreamingActivityProcessor): def __init__(self, settings: BotFrameworkAdapterSettings): - super().__init__(self, settings) + super().__init__(settings) self.connected_bot: Bot = None self.claims_identity: ClaimsIdentity = None @@ -57,7 +62,95 @@ async def process_streaming_activity( if self.claims_identity: context.turn_state[self.BOT_IDENTITY_KEY] = self.claims_identity - connector_client = self.create_connector_client() + connector_client = self._create_streaming_connector_client( + activity, request_handler + ) + context.turn_state[self.BOT_CONNECTOR_CLIENT_KEY] = connector_client + + await self.run_pipeline(context, bot_callback_handler) + + if activity.type == ActivityTypes.invoke: + activity_invoke_response = context.turn_state.get(self._INVOKE_RESPONSE_KEY) + + if not activity_invoke_response: + return InvokeResponse(status=HTTPStatus.NOT_IMPLEMENTED) + return activity_invoke_response.value + + return None + + async def send_streaming_activity(self, activity: Activity) -> ResourceResponse: + # Check to see if any of this adapter's StreamingRequestHandlers is associated with this conversation. + possible_handlers = [ + handler + for handler in self.request_handlers + if handler.service_url == activity.service_url + and handler.has_conversation(activity.conversation.id) + ] + + if possible_handlers: + if len(possible_handlers) > 1: + # The conversation has moved to a new connection and the former + # StreamingRequestHandler needs to be told to forget about it. + possible_handlers.sort( + key=lambda handler: handler.conversation_added_time( + activity.conversation.id + ) + ) + correct_handler = possible_handlers[-1] + for handler in possible_handlers: + if handler is not correct_handler: + handler.forget_conversation(activity.conversation.id) + + return await correct_handler.send_activity(activity) + + return await possible_handlers[0].send_activity(activity) + else: + if self.connected_bot: + # This is a proactive message that will need a new streaming connection opened. + # The ServiceUrl of a streaming connection follows the pattern "url:[ChannelName]:[Protocol]:[Host]". + + uri = activity.service_url.split(":") + protocol = uri[len(uri) - 2] + host = uri[len(uri) - 1] + # TODO: discuss if should abstract this from current package + # TODO: manage life cycle of sessions (when should we close them) + session = ClientSession() + aiohttp_ws = await session.ws_connect(protocol + host + "/api/messages") + web_socket = AiohttpWebSocket(aiohttp_ws, session) + handler = StreamingRequestHandler(self.connected_bot, self, web_socket) + + if self.request_handlers is None: + self.request_handlers = [] + + self.request_handlers.append(handler) + + return await handler.send_activity(activity) + + return None + + def can_process_outgoing_activity(self, activity: Activity) -> bool: + if not activity: + raise TypeError( + f"'activity: {activity.__class__.__name__}' argument can't be None" + ) + + return not activity.service_url.startswith("https") + + async def process_outgoing_activity( + self, turn_context: TurnContext, activity: Activity + ) -> ResourceResponse: + if not activity: + raise TypeError( + f"'activity: {activity.__class__.__name__}' argument can't be None" + ) + + # TODO: Check if we have token responses from OAuth cards. + + # The ServiceUrl for streaming channels begins with the string "urn" and contains + # information unique to streaming connections. Now that we know that this is a streaming + # activity, process it in the streaming pipeline. + # Process streaming activity. + return await self.send_streaming_activity(activity) def _create_streaming_connector_client( self, activity: Activity, request_handler: StreamingRequestHandler @@ -68,4 +161,12 @@ def _create_streaming_connector_client( else MicrosoftGovernmentAppCredentials.empty() ) - streaming_client + streaming_driver = StreamingHttpDriver(request_handler) + connector_client = ConnectorClient( + empty_credentials, + activity.service_url, + pipeline_type=AsyncBfPipeline, + driver=streaming_driver, + ) + + return connector_client diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index adbdd6044..6785fa8d1 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -3,13 +3,48 @@ from http import HTTPStatus from logging import Logger +from typing import Any, Optional -from botbuilder.streaming import StreamingRequest +from msrest.universal_http import ClientRequest +from msrest.universal_http.async_abc import AsyncClientResponse, AsyncHTTPSender +from botbuilder.streaming import StreamingRequest, ReceiveResponse from .streaming_request_handler import StreamingRequestHandler -class StreamingHttpClient: +class StreamingProtocolClientResponse(AsyncClientResponse): + def __init__( + self, request: StreamingRequest, streaming_response: ReceiveResponse + ) -> None: + super(StreamingProtocolClientResponse, self).__init__( + request, streaming_response + ) + # https://aiohttp.readthedocs.io/en/stable/client_reference.html#aiohttp.ClientResponse + self.status_code = streaming_response.status_code + # self.headers = streaming_response.headers + # self.reason = streaming_response.reason + self._body = None + + def body(self) -> bytes: + """Return the whole body as bytes in memory. + """ + if not self._body: + raise ValueError( + "Body is not available. Call async method load_body, or do your call with stream=False." + ) + return self._body + + async def load_body(self) -> None: + """Load in memory the body, so it could be accessible from sync methods.""" + self._body: ReceiveResponse + self._body = self.internal_response.read_body() + + def raise_for_status(self): + if 400 <= self.internal_response.status_code <= 599: + raise Exception(f"Http error: {self.internal_response.status_code}") + + +class StreamingHttpDriver(AsyncHTTPSender): def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = None): if not request_handler: raise TypeError( @@ -18,16 +53,24 @@ def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = No self._request_handler = request_handler self._logger = logger - async def send(self, request: object) -> object: + async def __aenter__(self): + raise Exception("This driver currently does not support context manager") + + async def __aexit__(self, *exc_details): # pylint: disable=arguments-differ + raise Exception("This driver currently does not support context manager") + + async def send(self, request: ClientRequest, **config: Any) -> AsyncClientResponse: # TODO: validate form of request to perform operations streaming_request = StreamingRequest( - path=request.path[request.path.index("/v3") :], verb=request.method + path=request.url[request.url.index("/v3") :], verb=request.method ) - streaming_request.set_body(request.content) + streaming_request.set_body(request.data) return await self._send_request(streaming_request) - async def _send_request(self, request: StreamingRequest) -> object: + async def _send_request( + self, request: StreamingRequest + ) -> StreamingProtocolClientResponse: try: server_response = await self._request_handler.send_streaming_request( request @@ -38,9 +81,11 @@ async def _send_request(self, request: StreamingRequest) -> object: if server_response.status_code == HTTPStatus.OK: # TODO: this should be an object read from json - return server_response.read_body_as_str() + + return StreamingProtocolClientResponse(request, server_response) except Exception as error: # TODO: log error + raise error pass return None diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py index d601123ff..1f4b998cb 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -1,26 +1,129 @@ -from aiohttp.web import Request, Response, WebSocketResponse -from botbuilder.core import BotFrameworkAdapter, Bot +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +from typing import Optional -class BotFrameworkHttpAdapter(BotFrameworkAdapter): - async def process(self, request: Request, ws_response: WebSocketResponse, bot: Bot): +from aiohttp.web import ( + Request, + Response, + json_response, + WebSocketResponse, + HTTPBadRequest, + HTTPUnauthorized, + HTTPUnsupportedMediaType +) +from botbuilder.core import Bot, BotFrameworkAdapterSettings +from botbuilder.core.streaming import ( + AiohttpWebSocket, + BotFrameworkHttpAdapterBase, + StreamingRequestHandler, +) +from botbuilder.schema import Activity +from botframework.connector.auth import AuthenticationConstants, JwtTokenValidation + + +class BotFrameworkHttpAdapter(BotFrameworkHttpAdapterBase): + def __init__(self, settings: BotFrameworkAdapterSettings): + super().__init__(settings) + + self._AUTH_HEADER_NAME = "authorization" + self._CHANNEL_ID_HEADER_NAME = "channelid" + + async def process(self, request: Request, ws_response: WebSocketResponse, bot: Bot) -> Optional[Response]: + # TODO: maybe it's not necessary to expose the ws_response if not request: raise TypeError("request can't be None") - if not ws_response: + if ws_response is None: raise TypeError("ws_response can't be None") if not bot: raise TypeError("bot can't be None") if request.method == "GET": - await self.connect_web_socket(bot, request, ws_response) + await self._connect_web_socket(bot, request, ws_response) + else: + # Deserialize the incoming Activity + if "application/json" in request.headers["Content-Type"]: + body = await request.json() + else: + raise HTTPUnsupportedMediaType() + + activity = Activity().deserialize(body) + auth_header = request.headers["Authorization"] if "Authorization" in request.headers else "" - async def connect_web_socket( + # Process the inbound activity with the bot + invoke_response = await self.process_activity(activity, auth_header, bot.on_turn) + if invoke_response: + return json_response(data=invoke_response.body, status=invoke_response.status) + return Response(status=201) + + async def _connect_web_socket( self, bot: Bot, request: Request, ws_response: WebSocketResponse ): if not request: raise TypeError("request can't be None") - if not ws_response: + if ws_response is None: raise TypeError("ws_response can't be None") + if not bot: + raise TypeError(f"'bot: {bot.__class__.__name__}' argument can't be None") + if not ws_response.can_prepare(request): - raise Exception("WS not available") + raise HTTPBadRequest(text="Upgrade to WebSocket is required.") + + if not await self._http_authenticate_request(request): + raise HTTPUnauthorized(text="Request authentication failed.") + + try: + await ws_response.prepare(request) + bf_web_socket = AiohttpWebSocket(ws_response) + request_handler = StreamingRequestHandler(bot, self, bf_web_socket) + + if self.request_handlers is None: + self.request_handlers = [] + + self.request_handlers.append(request_handler) + + await request_handler.listen() + except Exception as error: + raise Exception(f"Unable to create transport server. Error: {str(error)}") + + async def _http_authenticate_request(self, request: Request) -> bool: + try: + if not await self._credential_provider.is_authentication_disabled(): + auth_header = request.headers.get(self._AUTH_HEADER_NAME) + channel_id = request.headers.get(self._CHANNEL_ID_HEADER_NAME) + + if not auth_header: + await self._write_unauthorized_response(self._AUTH_HEADER_NAME) + return False + if not channel_id: + await self._write_unauthorized_response( + self._CHANNEL_ID_HEADER_NAME + ) + return False + + claims_identity = await JwtTokenValidation.validate_auth_header( + auth_header, + self._credential_provider, + self._channel_provider, + channel_id, + ) + + if not claims_identity.is_authenticated: + raise HTTPUnauthorized() + + # Add ServiceURL to the cache of trusted sites in order to allow token refreshing. + self._credentials.trust_service_url( + claims_identity.claims.get( + AuthenticationConstants.SERVICE_URL_CLAIM + ) + ) + self.claims_identity = claims_identity + return True + except Exception as error: + raise error + + async def _write_unauthorized_response(self, header_name: str): + raise HTTPUnauthorized( + text=f"Unable to authenticate. Missing header: {header_name}" + ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index f8449b357..368589c0e 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -6,16 +6,16 @@ # -------------------------------------------------------------------------- from .about import __version__, __title__ -from .protocol_adapter import ProtocolAdapter from .receive_request import ReceiveRequest +from .protocol_adapter import ProtocolAdapter from .receive_response import ReceiveResponse from .request_handler import RequestHandler from .streaming_request import StreamingRequest from .streaming_response import StreamingResponse __all__ = [ - "ProtocolAdapter", "ReceiveRequest", + "ProtocolAdapter", "ReceiveResponse", "RequestHandler", "StreamingRequest", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index a452c19f4..6a6a83468 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -5,6 +5,7 @@ from typing import List from threading import Lock +import botbuilder.streaming.payloads as payloads from botbuilder.streaming.payloads.models import Header from .assembler import Assembler @@ -13,14 +14,13 @@ class PayloadStreamAssembler(Assembler): def __init__( self, - stream_manager: "StreamManager", + stream_manager: "payloads.StreamManager", identifier: UUID, type: str = None, length: int = None, ): - from botbuilder.streaming.payloads import StreamManager - self._stream_manager = stream_manager or StreamManager() + self._stream_manager = stream_manager or payloads.StreamManager() self._stream: List[int] = [] self._lock = Lock() self.identifier = identifier diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py index 43631a8b7..dd1524985 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -6,8 +6,8 @@ from threading import Lock from typing import Awaitable, Callable, List -from botbuilder.streaming import ReceiveRequest -from botbuilder.streaming.payloads import ContentStream, StreamManager +import botbuilder.streaming as streaming +import botbuilder.streaming.payloads as payloads from botbuilder.streaming.payloads.models import Header, RequestPayload from .assembler import Assembler @@ -17,8 +17,8 @@ class ReceiveRequestAssembler(Assembler): def __init__( self, header: Header, - stream_manager: StreamManager, - on_completed: Callable[[UUID, ReceiveRequest], Awaitable], + stream_manager: "payloads.StreamManager", + on_completed: Callable[[UUID, "streaming.ReceiveRequest"], Awaitable], ): if not header: raise TypeError( @@ -57,7 +57,7 @@ def close(self): async def process_request(self, stream: List[int]): request_payload = RequestPayload().from_json(bytes(stream).decode("utf8")) - request = ReceiveRequest( + request = streaming.ReceiveRequest( verb=request_payload.verb, path=request_payload.path, streams=[] ) @@ -76,7 +76,7 @@ async def process_request(self, stream: List[int]): stream_assembler.content_type = stream_description.content_type stream_assembler.content_length = stream_description.length - content_stream = ContentStream( + content_stream = payloads.ContentStream( identifier=identifier, assembler=stream_assembler ) content_stream.length = stream_description.length diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py index 123fadbc4..a682456fa 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py @@ -6,8 +6,8 @@ from threading import Lock from typing import Awaitable, Callable, List -from botbuilder.streaming import ReceiveResponse -from botbuilder.streaming.payloads import ContentStream, StreamManager +import botbuilder.streaming as streaming +import botbuilder.streaming.payloads as payloads from botbuilder.streaming.payloads.models import Header, ResponsePayload from .assembler import Assembler @@ -17,8 +17,8 @@ class ReceiveResponseAssembler(Assembler): def __init__( self, header: Header, - stream_manager: StreamManager, - on_completed: Callable[[UUID, ReceiveResponse], Awaitable], + stream_manager: "payloads.StreamManager", + on_completed: Callable[[UUID, "payloads.StreamManager"], Awaitable], ): if not header: raise TypeError( @@ -57,7 +57,9 @@ def close(self): async def process_response(self, stream: List[int]): response_payload = ResponsePayload().from_json(bytes(stream).decode("utf8")) - response = ReceiveResponse(status_code=response_payload.status_code, streams=[]) + response = streaming.ReceiveResponse( + status_code=response_payload.status_code, streams=[] + ) if response_payload.streams: for stream_description in response_payload.streams: @@ -74,7 +76,7 @@ async def process_response(self, stream: List[int]): stream_assembler.content_type = stream_description.content_type stream_assembler.content_length = stream_description.length - content_stream = ContentStream( + content_stream = payloads.ContentStream( identifier=identifier, assembler=stream_assembler ) content_stream.length = stream_description.length diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py index a364d675a..065a29062 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py @@ -4,7 +4,7 @@ from uuid import UUID from typing import List -from botbuilder.streaming import StreamingRequest +import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.models import PayloadTypes, RequestPayload @@ -13,7 +13,10 @@ class RequestDisassembler(PayloadDisassembler): def __init__( - self, sender: PayloadSender, identifier: UUID, request: StreamingRequest + self, + sender: PayloadSender, + identifier: UUID, + request: "streaming.StreamingRequest", ): super().__init__(sender, identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py index 48c083996..6548c3f57 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py @@ -4,7 +4,7 @@ from uuid import UUID from typing import List -from botbuilder.streaming import StreamingResponse +import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.models import PayloadTypes, ResponsePayload @@ -13,7 +13,10 @@ class ResponseDisassembler(PayloadDisassembler): def __init__( - self, sender: PayloadSender, identifier: UUID, response: StreamingResponse + self, + sender: PayloadSender, + identifier: UUID, + response: "streaming.StreamingResponse", ): super().__init__(sender, identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py index 9a02de7f8..89f85f5c8 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py @@ -4,7 +4,7 @@ from uuid import UUID from typing import Awaitable, Callable, Dict, List -from botbuilder.streaming import ReceiveResponse, ReceiveRequest +import botbuilder.streaming as streaming from botbuilder.streaming.payloads.assemblers import ( Assembler, ReceiveRequestAssembler, @@ -19,8 +19,8 @@ class PayloadAssemblerManager: def __init__( self, stream_manager: StreamManager, - on_receive_request: Callable[[UUID, ReceiveRequest], Awaitable], - on_receive_response: Callable[[UUID, ReceiveResponse], Awaitable], + on_receive_request: Callable[[UUID, "streaming.ReceiveRequest"], Awaitable], + on_receive_response: Callable[[UUID, "streaming.ReceiveResponse"], Awaitable], ): self._on_receive_request = on_receive_request self._on_receive_response = on_receive_response diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py index 07d4e0ccb..c3138745d 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py @@ -5,15 +5,19 @@ from uuid import UUID from typing import Dict -from botbuilder.streaming import ReceiveResponse +import botbuilder.streaming as streaming class RequestManager: - def __init__(self, *, pending_requests: Dict[UUID, "Future[ReceiveResponse]"]): + def __init__( + self, + *, + pending_requests: Dict[UUID, "Future[streaming.ReceiveResponse]"] = None + ): self._pending_requests = pending_requests or {} async def signal_response( - self, request_id: UUID, response: ReceiveResponse + self, request_id: UUID, response: "streaming.ReceiveResponse" ) -> bool: # TODO: dive more into this logic signal: Future = self._pending_requests.get(request_id) @@ -26,7 +30,7 @@ async def signal_response( return False - async def get_response(self, request_id: UUID) -> ReceiveResponse: + async def get_response(self, request_id: UUID) -> "streaming.ReceiveResponse": if request_id in self._pending_requests: return None @@ -34,7 +38,7 @@ async def get_response(self, request_id: UUID) -> ReceiveResponse: self._pending_requests[request_id] = pending_request try: - response: ReceiveResponse = await pending_request + response: streaming.ReceiveResponse = await pending_request return response finally: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py index 1b94f6e08..c8ed122d9 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py @@ -4,7 +4,7 @@ import asyncio from uuid import UUID -from botbuilder.streaming import StreamingRequest, StreamingResponse +import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.disassemblers import ( CancelDisassembler, @@ -19,7 +19,9 @@ class SendOperations: def __init__(self, payload_sender: PayloadSender): self._payload_sender = payload_sender - async def send_request(self, identifier: UUID, request: StreamingRequest): + async def send_request( + self, identifier: UUID, request: "streaming.StreamingRequest" + ): disassembler = RequestDisassembler(self._payload_sender, identifier, request) await disassembler.disassemble() @@ -34,7 +36,9 @@ async def send_request(self, identifier: UUID, request: StreamingRequest): await asyncio.gather(*tasks) - async def send_response(self, identifier: UUID, response: StreamingResponse): + async def send_response( + self, identifier: UUID, response: "streaming.StreamingResponse" + ): disassembler = ResponseDisassembler(self._payload_sender, identifier, response) await disassembler.disassemble() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index 3a9211a5b..52641051f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -14,7 +14,7 @@ def __init__(self, status_code: int = None, streams: List[ContentStream] = None) self.streams = streams def read_body_as_json( - self, cls: Type[Model, Serializable] + self, cls: Union[Type[Model], Type[Serializable]] ) -> Union[Model, Serializable]: try: body_str = self.read_body_as_str() @@ -29,7 +29,7 @@ def read_body_as_json( def read_body_as_str(self) -> str: try: - content_stream = self.streams[0] if self.streams else None + content_stream = self.read_body() if not content_stream: return "" @@ -37,3 +37,14 @@ def read_body_as_str(self) -> str: return bytes(content_stream.stream).decode("utf8") except Exception as error: raise error + + def read_body(self) -> bytes: + try: + content_stream = self.streams[0] if self.streams else None + + if not content_stream: + return None + + return bytes(content_stream.stream) + except Exception as error: + raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py index a69923d0d..c52e5b95a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -57,17 +57,22 @@ def create_put(path: str = None, body: object = None) -> "StreamingRequest": def create_delete(path: str = None, body: object = None) -> "StreamingRequest": return StreamingRequest.create_request("DELETE", path, body) - def set_body(self, body: Union[str, Serializable, Model]): + def set_body(self, body: Union[str, Serializable, Model, bytes]): # TODO: verify if msrest.serialization.Model is necessary if not body: return - if isinstance(body, Serializable): - body = body.to_json() - elif isinstance(body, Model): - body = json.dumps(body.as_dict()) + if isinstance(body, bytes): + pass + else: + if isinstance(body, Serializable): + body = body.to_json() + elif isinstance(body, Model): + body = json.dumps(body.as_dict()) - self.add_stream(list(body.encode())) + body = body.encode("utf8") + + self.add_stream(list(body)) def add_stream(self, content: object, stream_id: UUID = None): if not content: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py index 335aa0808..ef5847cbf 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +from .web_socket import WebSocketMessage from .web_socket import WebSocket from .web_socket_close_status import WebSocketCloseStatus from .web_socket_server import WebSocketServer @@ -10,6 +10,7 @@ from .web_socket_state import WebSocketState __all__ = [ + "WebSocketMessage", "WebSocket", "WebSocketCloseStatus", "WebSocketMessageType", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py index 3b67ac1f7..e4a36b61c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py @@ -1,7 +1,35 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from abc import ABC +from typing import List, Any -class WebSocket: - def __init__(self): - self +from .web_socket_close_status import WebSocketCloseStatus +from .web_socket_state import WebSocketState +from .web_socket_message_type import WebSocketMessageType + + +class WebSocketMessage: + def __init__(self, *, message_type: WebSocketMessageType, data: List[int]): + self.message_type = message_type + self.data = data + + +class WebSocket(ABC): + def dispose(self): + raise NotImplementedError() + + async def close(self, close_status: WebSocketCloseStatus, status_description: str): + raise NotImplementedError() + + async def receive(self) -> WebSocketMessage: + raise NotImplementedError() + + async def send( + self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool + ): + raise NotImplementedError() + + @property + async def status(self) -> WebSocketState: + raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index 373e6abba..507f45b2a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from typing import List + from botbuilder.streaming.transport import TransportReceiverBase, TransportSenderBase from .web_socket import WebSocket @@ -40,12 +42,12 @@ async def close(self): # TODO: considering to create a BFTransportBuffer class to abstract the logic of binary buffers adapting to # current interfaces async def receive( - self, buffer: [object], offset: int = None, count: int = None + self, buffer: List[int], offset: int = None, count: int = None ) -> int: try: if self._socket: result = await self._socket.receive() - buffer.append(result) + buffer.extend(result.data) if result.message_type == WebSocketMessageType.CLOSE: await self._socket.close( WebSocketCloseStatus.NORMAL_CLOSURE, "Socket closed" @@ -55,7 +57,7 @@ async def receive( if self._socket.status == WebSocketState.CLOSED: self._socket.dispose() - return len(result) + return len(result.data) except Exception as error: # Exceptions of the three types below will also have set the socket's state to closed, which fires an # event consumers of this class are subscribed to and have handling around. Any other exception needs to @@ -64,7 +66,7 @@ async def receive( # TODO: might need to remove offset and count if no segmentation possible (or put them in BFTransportBuffer) async def send( - self, buffer: [object], offset: int = None, count: int = None + self, buffer: List[int], offset: int = None, count: int = None ) -> int: try: if self._socket: diff --git a/libraries/botframework-connector/botframework/connector/__init__.py b/libraries/botframework-connector/botframework/connector/__init__.py index 5dd822c31..a90857529 100644 --- a/libraries/botframework-connector/botframework/connector/__init__.py +++ b/libraries/botframework-connector/botframework/connector/__init__.py @@ -15,9 +15,9 @@ from .version import VERSION # TODO: Experimental -from .aiohttp_bf_pipeline import AiohttpBfPipeline +from .aiohttp_bf_pipeline import AsyncBfPipeline from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration -__all__ = ["AiohttpBfPipeline", "Channels", "ConnectorClient", "EmulatorApiClient"] +__all__ = ["AsyncBfPipeline", "Channels", "ConnectorClient", "EmulatorApiClient"] __version__ = VERSION diff --git a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py index 67b8ff2aa..4d2db8aea 100644 --- a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py +++ b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py @@ -11,10 +11,11 @@ from typing import Optional, Type -from msrest.async_client import SDKClientAsync, AsyncPipeline +from msrest.universal_http.async_abc import AsyncHTTPSender as AsyncHttpDriver +from msrest.pipeline.aiohttp import AsyncHTTPSender +from msrest.async_client import AsyncPipeline from msrest import Serializer, Deserializer -from .._configuration import ConnectorClientConfiguration from .operations_async import AttachmentsOperations from .operations_async import ConversationsOperations from .. import models @@ -63,11 +64,17 @@ def __init__( credentials, base_url=None, *, - pipeline_class: Optional[Type[AsyncPipeline]] = None + pipeline_type: Optional[Type[AsyncPipeline]] = None, + sender: Optional[AsyncHTTPSender] = None, + driver: Optional[AsyncHttpDriver] = None ): self.config = BotFrameworkConnectorConfiguration( - credentials, base_url, pipeline=pipeline_class + credentials, + base_url, + pipeline_type=pipeline_type, + sender=sender, + driver=driver, ) super(ConnectorClient, self).__init__(self.config) diff --git a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py index 151b4d7e4..26e46f061 100644 --- a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py +++ b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import asyncio -from msrest.async_client import ServiceClientAsync + from msrest.pipeline import AsyncPipeline, AsyncHTTPPolicy, SansIOHTTPPolicy from msrest.pipeline.aiohttp import AioHTTPSender from msrest.universal_http.aiohttp import AioHTTPSender as Driver @@ -12,7 +12,7 @@ from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration -class AiohttpBfPipeline(AsyncPipeline): +class AsyncBfPipeline(AsyncPipeline): def __init__(self, config: BotFrameworkConnectorConfiguration): creds = config.credentials @@ -28,7 +28,8 @@ def __init__(self, config: BotFrameworkConnectorConfiguration): # Assume this is the old credentials class, and then requests. Wrap it. policies.insert(1, AsyncRequestsCredentialsPolicy(creds)) - super().__init__(policies, AioHTTPSender(BFAioHTTPDriver)) + sender = config.sender or AioHTTPSender(config.driver or BFAioHTTPDriver) + super().__init__(policies, sender) class BFAioHTTPDriver(Driver): diff --git a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py index 7b92265d3..c24bb962c 100644 --- a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py +++ b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py @@ -4,7 +4,9 @@ from typing import Optional, Type from msrest.async_client import SDKClientAsync, ServiceClientAsync +from msrest.universal_http.async_abc import AsyncHTTPSender as AsyncHttpDriver from msrest.pipeline import AsyncPipeline +from msrest.pipeline.aiohttp import AsyncHTTPSender from ._configuration import ConnectorClientConfiguration @@ -16,12 +18,18 @@ def __init__( credentials, base_url: str, *, - pipeline: Optional[Type[AsyncPipeline]] = None + pipeline_type: Optional[Type[AsyncPipeline]] = None, + sender: Optional[AsyncHTTPSender] = None, + driver: Optional[AsyncHttpDriver] = None ): super().__init__(credentials, base_url) - if pipeline: - self.pipeline = pipeline(self) + # The overwrite hierarchy should be well documented + self.sender = sender + self.driver = driver + + if pipeline_type: + self.pipeline = pipeline_type(self) class BotFrameworkSDKClientAsync(SDKClientAsync): From e8ed7ebd99dc4082e5655b2143eeb91034d5d814 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 30 Apr 2020 12:08:38 -0700 Subject: [PATCH 11/37] Fix couple of errors, still in debugging phase. Initial receive doesnt work with current structure. --- .../core/streaming/aiohttp_web_socket.py | 20 ++++++---- .../streaming/streaming_request_handler.py | 1 + ...tp_channel_service_exception_middleware.py | 4 ++ .../aiohttp/bot_framework_http_adapter.py | 37 +++++++++++++++---- .../payload_transport/payload_receiver.py | 10 ++--- .../payload_transport/payload_sender.py | 4 +- .../streaming/payload_transport/send_queue.py | 8 ++-- .../transport/web_socket/web_socket_server.py | 2 +- .../web_socket/web_socket_transport.py | 4 +- .../connector/auth/emulator_validation.py | 5 +-- 10 files changed, 62 insertions(+), 33 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py index 94e6e54a7..85515374b 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py @@ -35,14 +35,18 @@ async def close(self, close_status: WebSocketCloseStatus, status_description: st ) async def receive(self) -> WebSocketMessage: - message = await self._aiohttp_ws.receive() + try: + # message = await self._aiohttp_ws.receive() - return WebSocketMessage( - message_type=WebSocketMessageType(int(message.type)), - data=list(str(message.data).encode("utf8")) - if message.type == WSMsgType.TEXT - else list(message.data), - ) + async for message in self._aiohttp_ws: + return WebSocketMessage( + message_type=WebSocketMessageType(int(message.type)), + data=list(str(message.data).encode("utf8")) + if message.type == WSMsgType.TEXT + else list(message.data), + ) + except Exception as error: + raise error async def send( self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool @@ -57,5 +61,5 @@ async def send( ) @property - async def status(self) -> WebSocketState: + def status(self) -> WebSocketState: return WebSocketState.CLOSED if self._aiohttp_ws.closed else WebSocketState.OPEN diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index 48af65c93..e94a7b18e 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -62,6 +62,7 @@ def service_url(self) -> str: async def listen(self): await self._server.start() + # TODO: log it def has_conversation(self, conversation_id: str) -> bool: diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/aiohttp_channel_service_exception_middleware.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/aiohttp_channel_service_exception_middleware.py index 7c5091121..40b0d105d 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/aiohttp_channel_service_exception_middleware.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/aiohttp_channel_service_exception_middleware.py @@ -3,6 +3,7 @@ from aiohttp.web import ( middleware, + HTTPError, HTTPNotImplemented, HTTPUnauthorized, HTTPNotFound, @@ -25,5 +26,8 @@ async def aiohttp_error_middleware(request, handler): raise HTTPUnauthorized() except KeyError: raise HTTPNotFound() + except HTTPError as error: + # In the case the integration adapter raises a specific HTTPError + raise error except Exception: raise HTTPInternalServerError() diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py index 1f4b998cb..a33c7dd9f 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -10,9 +10,9 @@ WebSocketResponse, HTTPBadRequest, HTTPUnauthorized, - HTTPUnsupportedMediaType + HTTPUnsupportedMediaType, ) -from botbuilder.core import Bot, BotFrameworkAdapterSettings +from botbuilder.core import Bot, BotFrameworkAdapterSettings, BotFrameworkAdapter from botbuilder.core.streaming import ( AiohttpWebSocket, BotFrameworkHttpAdapterBase, @@ -29,12 +29,14 @@ def __init__(self, settings: BotFrameworkAdapterSettings): self._AUTH_HEADER_NAME = "authorization" self._CHANNEL_ID_HEADER_NAME = "channelid" - async def process(self, request: Request, ws_response: WebSocketResponse, bot: Bot) -> Optional[Response]: + async def process( + self, request: Request, ws_response: WebSocketResponse, bot: Bot + ) -> Optional[Response]: # TODO: maybe it's not necessary to expose the ws_response if not request: raise TypeError("request can't be None") - if ws_response is None: - raise TypeError("ws_response can't be None") + #if ws_response is None: + #raise TypeError("ws_response can't be None") if not bot: raise TypeError("bot can't be None") @@ -48,12 +50,20 @@ async def process(self, request: Request, ws_response: WebSocketResponse, bot: B raise HTTPUnsupportedMediaType() activity = Activity().deserialize(body) - auth_header = request.headers["Authorization"] if "Authorization" in request.headers else "" + auth_header = ( + request.headers["Authorization"] + if "Authorization" in request.headers + else "" + ) # Process the inbound activity with the bot - invoke_response = await self.process_activity(activity, auth_header, bot.on_turn) + invoke_response = await self.process_activity( + activity, auth_header, bot.on_turn + ) if invoke_response: - return json_response(data=invoke_response.body, status=invoke_response.status) + return json_response( + data=invoke_response.body, status=invoke_response.status + ) return Response(status=201) async def _connect_web_socket( @@ -75,9 +85,13 @@ async def _connect_web_socket( try: await ws_response.prepare(request) + bf_web_socket = AiohttpWebSocket(ws_response) + request_handler = StreamingRequestHandler(bot, self, bf_web_socket) + # await request_handler._server._receiver.connect(request_handler._server._web_socket_transport) + if self.request_handlers is None: self.request_handlers = [] @@ -85,6 +99,8 @@ async def _connect_web_socket( await request_handler.listen() except Exception as error: + import traceback + traceback.print_exc() raise Exception(f"Unable to create transport server. Error: {str(error)}") async def _http_authenticate_request(self, request: Request) -> bool: @@ -112,6 +128,11 @@ async def _http_authenticate_request(self, request: Request) -> bool: if not claims_identity.is_authenticated: raise HTTPUnauthorized() + self._credentials = self._credentials or await self._BotFrameworkAdapter__get_app_credentials( + self.settings.app_id, + AuthenticationConstants.TO_CHANNEL_FROM_BOT_OAUTH_SCOPE, + ) + # Add ServiceURL to the cache of trusted sites in order to allow token refreshing. self._credentials.trust_service_url( claims_identity.claims.get( diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index c7b72604f..04d1998c2 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -31,17 +31,17 @@ def __init__(self): @property def is_connected(self) -> bool: - return self._receiver is None + return self._receiver is not None - def connect(self, receiver: TransportReceiverBase): + async def connect(self, receiver: TransportReceiverBase): if self._receiver: raise RuntimeError(f"{self.__class__.__name__} instance already connected.") self._receiver = receiver - self._connected_event.set() + await self._run_receive() - def _run_receive(self): - asyncio.create_task(self._receive_packets()) + async def _run_receive(self): + await self._receive_packets() def subscribe( self, diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index 3f9ef399e..718b0a42b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -19,7 +19,7 @@ # TODO: consider interface this class class PayloadSender: - def __init__(self,): + def __init__(self): self._connected_event = Event() self._sender: TransportSenderBase = None self._is_disconnecting: bool = False @@ -36,7 +36,7 @@ def __init__(self,): @property def is_connected(self) -> bool: - return self._sender is None + return self._sender is not None def connect(self, sender: TransportSenderBase): if self._sender: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index e089d93f5..5cfc5789b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -17,15 +17,15 @@ def __init__(self, action: Callable[[object], Awaitable], timeout: int = 30): # TODO: this have to be abstracted so can remove asyncio dependency loop = asyncio.get_event_loop() - loop.create_task(self._process) + loop.create_task(self._process()) def post(self, item: object): - self.post_internal(item) + self._post_internal(item) - def post_internal(self, item: object): + def _post_internal(self, item: object): self._queue.put(item) - async def process(self): + async def _process(self): while True: try: while True: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index bd26321ae..a864fb38c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -53,7 +53,7 @@ def is_connected(self) -> bool: async def start(self): self._closed_signal = Future() self._sender.connect(self._web_socket_transport) - self._receiver.connect(self._web_socket_transport) + await self._receiver.connect(self._web_socket_transport) return self._closed_signal diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index 507f45b2a..b73cf9a87 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -19,11 +19,11 @@ def __init__(self, web_socket: WebSocket): def is_connected(self): print("Getting value") # TODO: mock logic - return self._socket.status == "Open" + return self._socket.status == WebSocketState.OPEN async def close(self): # TODO: mock logic - if self._socket.status == "Open": + if self._socket.status == WebSocketState.OPEN: try: await self._socket.close( WebSocketCloseStatus.NORMAL_CLOSURE, diff --git a/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py b/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py index 0e2d7fcaa..86923bee3 100644 --- a/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py +++ b/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py @@ -177,9 +177,8 @@ async def authenticate_emulator_token( "Unauthorized. Unknown Emulator Token version ", version_claim, "." ) - is_valid_app_id = await asyncio.ensure_future( - credentials.is_valid_appid(app_id) - ) + is_valid_app_id = await credentials.is_valid_appid(app_id) + if not is_valid_app_id: raise PermissionError( "Unauthorized. Invalid AppId passed on token: ", app_id From 09062877f8d8e859060fb328d9f946bb2fa655fc Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Mon, 4 May 2020 22:27:43 -0700 Subject: [PATCH 12/37] Several fixes including deadlock in threading, serialization and minor logic bugs. PayloadStream logic pending. --- libraries/botbuilder-ai/setup.py | 2 +- .../core/streaming/aiohttp_web_socket.py | 16 +++++++------- .../aiohttp/bot_framework_http_adapter.py | 16 ++++++++------ .../aiohttp/bot_framework_http_client.py | 2 +- .../botbuilder-integration-aiohttp/setup.py | 2 +- .../setup.py | 2 +- .../botbuilder/streaming/payload_stream.py | 19 ++++++++++++++++ .../payload_transport/payload_receiver.py | 6 ++--- .../streaming/payload_transport/send_queue.py | 18 ++++++++++----- .../assemblers/payload_stream_assembler.py | 6 ++--- .../assemblers/receive_request_assembler.py | 4 ++-- .../streaming/payloads/header_serializer.py | 22 +++++++++++++------ .../streaming/payloads/models/header.py | 4 ++-- .../payloads/payload_assembler_manager.py | 2 +- .../botbuilder/streaming/receive_request.py | 1 + .../botbuilder/streaming/receive_response.py | 1 + .../botbuilder/streaming/streaming_request.py | 2 +- .../transport/web_socket/web_socket.py | 2 +- .../transport/web_socket/web_socket_server.py | 2 +- .../web_socket/web_socket_transport.py | 13 +++++++++-- 20 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py diff --git a/libraries/botbuilder-ai/setup.py b/libraries/botbuilder-ai/setup.py index 72f112a5a..85dbe51f7 100644 --- a/libraries/botbuilder-ai/setup.py +++ b/libraries/botbuilder-ai/setup.py @@ -8,7 +8,7 @@ "azure-cognitiveservices-language-luis==0.2.0", "botbuilder-schema>=4.7.1", "botbuilder-core>=4.7.1", - "aiohttp==3.6.2", + "aiohttp>=3.6.2", ] TESTS_REQUIRES = ["aiounittest>=1.1.0"] diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py index 85515374b..e96d22df1 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py @@ -36,15 +36,15 @@ async def close(self, close_status: WebSocketCloseStatus, status_description: st async def receive(self) -> WebSocketMessage: try: - # message = await self._aiohttp_ws.receive() + message = await self._aiohttp_ws.receive() - async for message in self._aiohttp_ws: - return WebSocketMessage( - message_type=WebSocketMessageType(int(message.type)), - data=list(str(message.data).encode("utf8")) - if message.type == WSMsgType.TEXT - else list(message.data), - ) + # async for message in self._aiohttp_ws: + return WebSocketMessage( + message_type=WebSocketMessageType(int(message.type)), + data=list(str(message.data).encode("ascii")) + if message.type == WSMsgType.TEXT + else list(message.data), + ) except Exception as error: raise error diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py index a33c7dd9f..0ce0905c8 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -35,8 +35,8 @@ async def process( # TODO: maybe it's not necessary to expose the ws_response if not request: raise TypeError("request can't be None") - #if ws_response is None: - #raise TypeError("ws_response can't be None") + # if ws_response is None: + # raise TypeError("ws_response can't be None") if not bot: raise TypeError("bot can't be None") @@ -90,8 +90,6 @@ async def _connect_web_socket( request_handler = StreamingRequestHandler(bot, self, bf_web_socket) - # await request_handler._server._receiver.connect(request_handler._server._web_socket_transport) - if self.request_handlers is None: self.request_handlers = [] @@ -100,6 +98,7 @@ async def _connect_web_socket( await request_handler.listen() except Exception as error: import traceback + traceback.print_exc() raise Exception(f"Unable to create transport server. Error: {str(error)}") @@ -128,9 +127,12 @@ async def _http_authenticate_request(self, request: Request) -> bool: if not claims_identity.is_authenticated: raise HTTPUnauthorized() - self._credentials = self._credentials or await self._BotFrameworkAdapter__get_app_credentials( - self.settings.app_id, - AuthenticationConstants.TO_CHANNEL_FROM_BOT_OAUTH_SCOPE, + self._credentials = ( + self._credentials + or await self._BotFrameworkAdapter__get_app_credentials( + self.settings.app_id, + AuthenticationConstants.TO_CHANNEL_FROM_BOT_OAUTH_SCOPE, + ) ) # Add ServiceURL to the cache of trusted sites in order to allow token refreshing. diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_client.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_client.py index 3fa2f448d..97dad2109 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_client.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_client.py @@ -109,7 +109,7 @@ async def post_activity( json_content = json.dumps(activity.serialize()) resp = await self._session.post( - to_url, data=json_content.encode("utf-8"), headers=headers_dict, + to_url, data=json_content.encode("utf8"), headers=headers_dict, ) resp.raise_for_status() data = (await resp.read()).decode() diff --git a/libraries/botbuilder-integration-aiohttp/setup.py b/libraries/botbuilder-integration-aiohttp/setup.py index df1778810..e0c8d3de7 100644 --- a/libraries/botbuilder-integration-aiohttp/setup.py +++ b/libraries/botbuilder-integration-aiohttp/setup.py @@ -9,7 +9,7 @@ "botbuilder-schema>=4.7.1", "botframework-connector>=4.7.1", "botbuilder-core>=4.7.1", - "aiohttp==3.6.2", + "aiohttp>=3.6.2", ] root = os.path.abspath(os.path.dirname(__file__)) diff --git a/libraries/botbuilder-integration-applicationinsights-aiohttp/setup.py b/libraries/botbuilder-integration-applicationinsights-aiohttp/setup.py index ea8c2f359..00fdd0e9c 100644 --- a/libraries/botbuilder-integration-applicationinsights-aiohttp/setup.py +++ b/libraries/botbuilder-integration-applicationinsights-aiohttp/setup.py @@ -6,7 +6,7 @@ REQUIRES = [ "applicationinsights>=0.11.9", - "aiohttp==3.6.2", + "aiohttp>=3.6.2", "botbuilder-schema>=4.7.1", "botframework-connector>=4.7.1", "botbuilder-core>=4.7.1", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py new file mode 100644 index 000000000..e2fbce84e --- /dev/null +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from threading import Lock +from typing import List + +from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler + + +class PayloadStream: + def __init__(self, assembler: PayloadStreamAssembler): + self._assembler = assembler + self._buffer_queue: List[int] = [] + self._lock = Lock() + self._producer_length = 0 # total length + self._consumer_length = 0 # read position + self._active: List[int] = [] + self._active_offset = 0 + self._end = False diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index 04d1998c2..0b2a4e89a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -128,7 +128,7 @@ async def _receive_packets(self): "TransportDisconnectedException: Stream closed while reading header bytes" ) - if content_stream: + if content_stream is not None: # write chunks to the content_stream if it's not a stream type # TODO: this has to be improved in custom buffer class (validate buffer ended) if not PayloadTypes.is_stream(header): @@ -138,13 +138,13 @@ async def _receive_packets(self): offset += length # give the full payload buffer to the contentStream if it's a stream - if content_stream and PayloadTypes.is_stream(header): + if content_stream is not None and PayloadTypes.is_stream(header): # TODO: should this be a copy? content_stream = buffer self._receive_action(header, content_stream, offset) except Exception as exception: is_closed = True - disconnect_args = DisconnectedEventArgs(reason=exception.message) + disconnect_args = DisconnectedEventArgs(reason=str(exception)) self.disconnect(disconnect_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index 5cfc5789b..7c5bde2b2 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -1,11 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import traceback + from queue import Queue from typing import Awaitable, Callable from threading import Event, Lock, Semaphore import asyncio +import threading class SendQueue: @@ -16,8 +19,13 @@ def __init__(self, action: Callable[[object], Awaitable], timeout: int = 30): self._timeout_seconds = timeout # TODO: this have to be abstracted so can remove asyncio dependency - loop = asyncio.get_event_loop() - loop.create_task(self._process()) + def schedule_task(): + loop = asyncio.new_event_loop() + loop.run_until_complete(self._process()) + + new_thread = threading.Thread(target=schedule_task, args=()) + new_thread.daemon = True + new_thread.start() def post(self, item: object): self._post_internal(item) @@ -29,9 +37,8 @@ async def _process(self): while True: try: while True: - item = self._queue.get() - if not item: - break + await asyncio.sleep(1) + item = self._queue.get(block=False) try: await self._action(item) except Exception: @@ -41,4 +48,5 @@ async def _process(self): self._queue.task_done() except Exception: # AppInsights.TrackException(e) + # traceback.print_exc() pass diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index 6a6a83468..656bec8b5 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -28,8 +28,8 @@ def __init__( self.content_length = length self.end: bool = None - # TODO: highly probable this can be removed def create_stream_from_payload(self) -> List[int]: + # TODO: return PayloadStream(self) return [] # TODO: somewhat probable this can be removed @@ -40,9 +40,7 @@ def get_payload_as_stream(self) -> List[int]: return self._stream - def on_receive( - self, header: Header, stream: List[int], content_length: int - ) -> List[int]: + def on_receive(self, header: Header, stream: List[int], content_length: int): if header.end: self.end = True diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py index dd1524985..c63c1423b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -55,7 +55,7 @@ def close(self): self._stream_manager.close_stream(self.identifier) async def process_request(self, stream: List[int]): - request_payload = RequestPayload().from_json(bytes(stream).decode("utf8")) + request_payload = RequestPayload().from_json(bytes(stream).decode("utf-8-sig")) request = streaming.ReceiveRequest( verb=request_payload.verb, path=request_payload.path, streams=[] @@ -64,7 +64,7 @@ async def process_request(self, stream: List[int]): if request_payload.streams: for stream_description in request_payload.streams: try: - identifier = UUID(int=int(stream_description.id)) + identifier = UUID(stream_description.id) except Exception: raise ValueError( f"Stream description id '{stream_description.id}' is not a Guid" diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py index faa0954e3..7ed704be4 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py @@ -80,7 +80,10 @@ def deserialize(buffer: List[int], offset: int, count: int) -> Header: raise ValueError("Header type delimeter is malformed") length_str = HeaderSerializer._binary_array_to_str( - buffer[HeaderSerializer.LENGTH_OFFSET : HeaderSerializer.LENGTH_LENGTH] + buffer[ + HeaderSerializer.LENGTH_OFFSET : HeaderSerializer.LENGTH_OFFSET + + HeaderSerializer.LENGTH_LENGTH + ] ) try: @@ -97,11 +100,14 @@ def deserialize(buffer: List[int], offset: int, count: int) -> Header: raise ValueError("Header length delimeter is malformed") identifier_str = HeaderSerializer._binary_array_to_str( - buffer[HeaderSerializer.ID_OFFSET : HeaderSerializer.ID_LENGTH] + buffer[ + HeaderSerializer.ID_OFFSET : HeaderSerializer.ID_OFFSET + + HeaderSerializer.ID_LENGTH + ] ) try: - identifier = UUID(int=int(identifier_str)) + identifier = UUID(identifier_str) except Exception: raise ValueError("Header id is malformed") @@ -116,6 +122,8 @@ def deserialize(buffer: List[int], offset: int, count: int) -> Header: ]: raise ValueError("Header end is malformed") + header.end = buffer[HeaderSerializer.END_OFFSET] == HeaderSerializer.END + if buffer[HeaderSerializer.TERMINATOR_OFFSET] != HeaderSerializer.TERMINATOR: raise ValueError("Header terminator is malformed") @@ -135,19 +143,19 @@ def _char_to_binary_int(char: str) -> int: @staticmethod def _int_to_formatted_encoded_str(value: int, str_format: str) -> bytes: - return str_format.format(value).encode() + return str_format.format(value).encode("ascii") @staticmethod def _uuid_to_numeric_encoded_str(value: UUID) -> bytes: - return str(int(value)).encode() + return str(int(value)).encode("ascii") @staticmethod def _binary_int_to_char(binary_int: int) -> str: - return bytes([binary_int]).decode("utf8") + return bytes([binary_int]).decode("ascii") @staticmethod def _binary_array_to_str(binary_array: List[int]) -> str: - return bytes(binary_array).decode("utf8") + return bytes(binary_array).decode("ascii") @staticmethod def _write_in_buffer(data: List[int], buffer: List[int], insert_index: int): diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py index fd9e24d88..c07d4d4bb 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py @@ -25,8 +25,8 @@ def payload_length(self, value: int): self._internal_payload_length = value def _validate_length(self, value: int, max_val: int, min_val: int): - if value > max: + if value > max_val: raise ValueError(f"Length must be less or equal than {max_val}") - if value < min: + if value < min_val: raise ValueError(f"Length must be greater or equal than {min_val}") diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py index 89f85f5c8..5ea9470bb 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py @@ -68,4 +68,4 @@ def _create_payload_assembler(self, header: Header) -> Assembler: return None def _is_stream_payload(self, header: Header) -> bool: - return header.type == PayloadTypes + return PayloadTypes.is_stream(header) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py index 73cc38ac2..39e1c25e6 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py @@ -21,6 +21,7 @@ def read_body_as_str(self) -> str: if not content_stream: return "" + # TODO: encoding double check return bytes(content_stream.stream).decode("utf8") except Exception as error: raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index 52641051f..f7680b464 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -34,6 +34,7 @@ def read_body_as_str(self) -> str: if not content_stream: return "" + # TODO: encoding double check return bytes(content_stream.stream).decode("utf8") except Exception as error: raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py index c52e5b95a..2e4611d88 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py @@ -70,7 +70,7 @@ def set_body(self, body: Union[str, Serializable, Model, bytes]): elif isinstance(body, Model): body = json.dumps(body.as_dict()) - body = body.encode("utf8") + body = body.encode("ascii") self.add_stream(list(body)) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py index e4a36b61c..c50cc1181 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py @@ -31,5 +31,5 @@ async def send( raise NotImplementedError() @property - async def status(self) -> WebSocketState: + def status(self) -> WebSocketState: raise NotImplementedError() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index a864fb38c..112af160b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -20,7 +20,7 @@ class WebSocketServer: def __init__(self, socket: WebSocket, request_handler: RequestHandler): - if not socket: + if socket is None: raise TypeError( f"'socket: {socket.__class__.__name__}' argument can't be None" ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index b73cf9a87..64f7e80ab 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -47,7 +47,16 @@ async def receive( try: if self._socket: result = await self._socket.receive() - buffer.extend(result.data) + buffer_index = buffer.index(None) if None in buffer else 0 + result_index = 0 + while ( + buffer_index < len(buffer) + and result_index < len(result.data) + and result_index < count + ): + buffer[buffer_index] = result.data[result_index] + buffer_index += 1 + result_index += 1 if result.message_type == WebSocketMessageType.CLOSE: await self._socket.close( WebSocketCloseStatus.NORMAL_CLOSURE, "Socket closed" @@ -57,7 +66,7 @@ async def receive( if self._socket.status == WebSocketState.CLOSED: self._socket.dispose() - return len(result.data) + return len(result.data) except Exception as error: # Exceptions of the three types below will also have set the socket's state to closed, which fires an # event consumers of this class are subscribed to and have handling around. Any other exception needs to From 5a4f8c33d0ad40cb4e4108782542f61a03bb26d8 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 12 May 2020 17:34:16 -0700 Subject: [PATCH 13/37] More errors fixed, trying to fit websocket into ms rest pipeline. Receiving is working with some bugs. --- .../bot_framework_http_adapter_base.py | 7 ++- .../core/streaming/streaming_http_client.py | 25 +++++--- .../streaming/streaming_request_handler.py | 6 +- .../botbuilder/streaming/__init__.py | 2 + .../botbuilder/streaming/payload_stream.py | 62 ++++++++++++++++++- .../payload_transport/payload_receiver.py | 9 +-- .../payload_transport/payload_sender.py | 2 +- .../assemblers/payload_stream_assembler.py | 14 ++--- .../assemblers/receive_request_assembler.py | 12 +++- .../assemblers/receive_response_assembler.py | 13 +++- .../payloads/models/stream_description.py | 4 +- .../payloads/payload_assembler_manager.py | 7 ++- .../streaming/payloads/stream_manager.py | 3 +- .../botbuilder/streaming/receive_request.py | 4 +- .../botframework/connector/__init__.py | 8 ++- .../connector/aio/_connector_client_async.py | 21 ++++--- .../connector/aiohttp_bf_pipeline.py | 20 +++--- .../bot_framework_sdk_client_async.py | 13 ++-- 18 files changed, 159 insertions(+), 73 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index 31a5829df..ac0b55954 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -13,7 +13,7 @@ TurnContext, ) from botbuilder.schema import Activity, ActivityTypes, ResourceResponse -from botframework.connector import AsyncBfPipeline +from botframework.connector import AsyncBfPipeline, BotFrameworkConnectorConfiguration from botframework.connector.aio import ConnectorClient from botframework.connector.auth import ( ClaimsIdentity, @@ -160,13 +160,14 @@ def _create_streaming_connector_client( if self._channel_provider and self._channel_provider.is_government() else MicrosoftGovernmentAppCredentials.empty() ) - streaming_driver = StreamingHttpDriver(request_handler) - connector_client = ConnectorClient( + config = BotFrameworkConnectorConfiguration( empty_credentials, activity.service_url, pipeline_type=AsyncBfPipeline, driver=streaming_driver, ) + streaming_driver.config = config + connector_client = ConnectorClient(None, custom_configuration=config) return connector_client diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index 6785fa8d1..5b2f91692 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -1,12 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import asyncio from http import HTTPStatus from logging import Logger -from typing import Any, Optional +from typing import Any from msrest.universal_http import ClientRequest -from msrest.universal_http.async_abc import AsyncClientResponse, AsyncHTTPSender +from msrest.universal_http.async_abc import AsyncClientResponse +from msrest.universal_http.async_requests import ( + AsyncRequestsHTTPSender as AsyncRequestsHTTPDriver, +) from botbuilder.streaming import StreamingRequest, ReceiveResponse from .streaming_request_handler import StreamingRequestHandler @@ -44,8 +48,15 @@ def raise_for_status(self): raise Exception(f"Http error: {self.internal_response.status_code}") -class StreamingHttpDriver(AsyncHTTPSender): - def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = None): +class StreamingHttpDriver(AsyncRequestsHTTPDriver): + def __init__( + self, + request_handler: StreamingRequestHandler, + *, + config=None, + logger: Logger = None, + ): + super().__init__(config) if not request_handler: raise TypeError( f"'request_handler: {request_handler.__class__.__name__}' argument can't be None" @@ -53,12 +64,6 @@ def __init__(self, request_handler: StreamingRequestHandler, logger: Logger = No self._request_handler = request_handler self._logger = logger - async def __aenter__(self): - raise Exception("This driver currently does not support context manager") - - async def __aexit__(self, *exc_details): # pylint: disable=arguments-differ - raise Exception("This driver currently does not support context manager") - async def send(self, request: ClientRequest, **config: Any) -> AsyncClientResponse: # TODO: validate form of request to perform operations streaming_request = StreamingRequest( diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index e94a7b18e..a6db1007f 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -5,6 +5,7 @@ from http import HTTPStatus from datetime import datetime from logging import Logger +from json import loads from typing import Dict, List from botbuilder.core import Bot @@ -90,7 +91,7 @@ async def process_request( # Convert the StreamingRequest into an activity the adapter can understand. try: - body = request.read_body_as_str() + body_str = request.read_body_as_str() except Exception as error: response.status_code = int(HTTPStatus.BAD_REQUEST) # TODO: log error @@ -99,7 +100,8 @@ async def process_request( try: # TODO: validate if should use deserialize or from_dict - activity: Activity = Activity.deserialize(body) + body_dict = loads(body_str) + activity: Activity = Activity.deserialize(body_dict) # All activities received by this StreamingRequestHandler will originate from the same channel, but we won't # know what that channel is until we've received the first request. diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py index 368589c0e..fac150fb5 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py @@ -7,6 +7,7 @@ from .about import __version__, __title__ from .receive_request import ReceiveRequest +from .payload_stream import PayloadStream from .protocol_adapter import ProtocolAdapter from .receive_response import ReceiveResponse from .request_handler import RequestHandler @@ -17,6 +18,7 @@ "ReceiveRequest", "ProtocolAdapter", "ReceiveResponse", + "PayloadStream", "RequestHandler", "StreamingRequest", "StreamingResponse", diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py index e2fbce84e..313957787 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from threading import Lock +from threading import Lock, Semaphore from typing import List from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler @@ -10,10 +10,66 @@ class PayloadStream: def __init__(self, assembler: PayloadStreamAssembler): self._assembler = assembler - self._buffer_queue: List[int] = [] + self._buffer: List[int] = [] self._lock = Lock() + self._data_available = Semaphore(0) self._producer_length = 0 # total length - self._consumer_length = 0 # read position + self._consumer_position = 0 # read position self._active: List[int] = [] self._active_offset = 0 self._end = False + + def __len__(self): + return len(self._buffer) + + def give_buffer(self, buffer: List[int]): + self._buffer.extend(buffer) + + self._data_available.release() + + def done_producing(self): + self.give_buffer([]) + + def write(self, buffer: List[int], offset: int, count: int): + buffer_copy = buffer[offset : offset + count] + self.give_buffer(buffer_copy) + + def read(self, buffer: List[int], offset: int, count: int): + if self._end: + return 0 + + if not self._active: + self._data_available.acquire() + with self._lock: + self._active.extend(self._buffer) + self._buffer = [] + + available_count = min(len(self._active) - self._active_offset, count) + + for index in range(available_count): + buffer[offset + index] = self._active[self._active_offset] + self._active_offset += 1 + + self._consumer_position += available_count + + if self._active_offset >= len(self._active): + self._active = [] + self._active_offset = 0 + + if ( + self._assembler + and self._consumer_position >= self._assembler.content_length + ): + self._end = True + + return available_count + + def read_until_end(self): + result = [None] * self._assembler.content_length + current_size = 0 + + while not self._end: + count = self.read(result, current_size, self._assembler.content_length) + current_size += count + + return result diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index 0b2a4e89a..c932f89b9 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import asyncio from typing import Callable, List +import botbuilder.streaming as streaming from botbuilder.streaming.payloads import HeaderSerializer from botbuilder.streaming.payloads.models import Header, PayloadTypes from botbuilder.streaming.transport import ( @@ -138,9 +138,10 @@ async def _receive_packets(self): offset += length # give the full payload buffer to the contentStream if it's a stream - if content_stream is not None and PayloadTypes.is_stream(header): - # TODO: should this be a copy? - content_stream = buffer + if PayloadTypes.is_stream(header) and isinstance( + content_stream, streaming.PayloadStream + ): + content_stream.give_buffer(buffer) self._receive_action(header, content_stream, offset) except Exception as exception: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index 718b0a42b..69f31c7d3 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -145,5 +145,5 @@ async def _write_packet(self, packet: SendPacket): # TODO: should this really run in the background? asyncio.create_task(packet.sent_callback(packet.header)) except Exception as exception: - disconnected_args = DisconnectedEventArgs(reason=exception.message) + disconnected_args = DisconnectedEventArgs(reason=str(exception)) self.disconnect(disconnected_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index 656bec8b5..434b02770 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -5,6 +5,7 @@ from typing import List from threading import Lock +import botbuilder.streaming as streaming import botbuilder.streaming.payloads as payloads from botbuilder.streaming.payloads.models import Header @@ -21,21 +22,19 @@ def __init__( ): self._stream_manager = stream_manager or payloads.StreamManager() - self._stream: List[int] = [] + self._stream: "streaming.PayloadStream" = None self._lock = Lock() self.identifier = identifier self.content_type = type self.content_length = length self.end: bool = None - def create_stream_from_payload(self) -> List[int]: - # TODO: return PayloadStream(self) - return [] + def create_stream_from_payload(self) -> "streaming.PayloadStream": + return streaming.PayloadStream(self) - # TODO: somewhat probable this can be removed - def get_payload_as_stream(self) -> List[int]: + def get_payload_as_stream(self) -> "streaming.PayloadStream": with self._lock: - if self._stream is None: + if not self._stream: self._stream = self.create_stream_from_payload() return self._stream @@ -43,6 +42,7 @@ def get_payload_as_stream(self) -> List[int]: def on_receive(self, header: Header, stream: List[int], content_length: int): if header.end: self.end = True + self._stream.done_producing() def close(self): self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py index c63c1423b..926e99f46 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -3,7 +3,7 @@ import asyncio from uuid import UUID -from threading import Lock +from threading import Lock, Thread from typing import Awaitable, Callable, List import botbuilder.streaming as streaming @@ -48,8 +48,14 @@ def on_receive(self, header: Header, stream: List[int], content_length: int): if header.end: self.end = True - # Execute the request on a separate Task - asyncio.create_task(self.process_request(stream)) + # Execute the request on a separate Thread in the background + def schedule_task(): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.process_request(stream)) + + new_thread = Thread(target=schedule_task, args=()) + new_thread.daemon = True + new_thread.start() def close(self): self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py index a682456fa..75363eb8a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py @@ -3,7 +3,7 @@ import asyncio from uuid import UUID -from threading import Lock +from threading import Lock, Thread from typing import Awaitable, Callable, List import botbuilder.streaming as streaming @@ -18,7 +18,7 @@ def __init__( self, header: Header, stream_manager: "payloads.StreamManager", - on_completed: Callable[[UUID, "payloads.StreamManager"], Awaitable], + on_completed: Callable[[UUID, "streaming.ReceiveResponse"], Awaitable], ): if not header: raise TypeError( @@ -49,7 +49,14 @@ def on_receive(self, header: Header, stream: List[int], content_length: int): self.end = header.end # Execute the response on a separate Task - asyncio.create_task(self.process_response(stream)) + # Execute the request on a separate Thread in the background + def schedule_task(): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.process_response(stream)) + + new_thread = Thread(target=schedule_task, args=()) + new_thread.daemon = True + new_thread.start() def close(self): self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py index 395ef4b47..154520f79 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py @@ -13,7 +13,7 @@ def __init__(self, *, id: str = None, content_type: str = None, length: int = No self.length = length def to_dict(self) -> dict: - obj = {"id": self.id, "contentType": self.content_type} + obj = {"id": self.id, "type": self.content_type} if self.length is not None: obj["length"] = self.length @@ -22,7 +22,7 @@ def to_dict(self) -> dict: def from_dict(self, json_dict: dict) -> "StreamDescription": self.id = json_dict.get("id") - self.content_type = json_dict.get("contentType") + self.content_type = json_dict.get("type") self.length = json_dict.get("length") return self diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py index 5ea9470bb..81d5fe07f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. from uuid import UUID -from typing import Awaitable, Callable, Dict, List +from typing import Awaitable, Callable, Dict, List, Union import botbuilder.streaming as streaming from botbuilder.streaming.payloads.assemblers import ( @@ -27,7 +27,10 @@ def __init__( self._stream_manager = stream_manager self._active_assemblers: Dict[UUID, Assembler] = {} - def get_payload_stream(self, header: Header) -> List[int]: + def get_payload_stream( + self, header: Header + ) -> Union[List[int], "streaming.PayloadStream"]: + # TODO: The return value SHOULDN'T be a union, we should interface List[int] into a BFStream class if self._is_stream_payload(header): return self._stream_manager.get_payload_stream(header) elif not self._active_assemblers.get(header.id): diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py index e58b5ee13..a9b4a0d83 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py @@ -4,6 +4,7 @@ from uuid import UUID from typing import Callable, Dict, List +import botbuilder.streaming as streaming from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler from botbuilder.streaming.payloads.models import Header @@ -22,7 +23,7 @@ def get_payload_assembler(self, identifier: UUID) -> PayloadStreamAssembler: return self._active_assemblers[identifier] - def get_payload_stream(self, header: Header) -> List[int]: + def get_payload_stream(self, header: Header) -> "streaming.PayloadStream": assembler = self.get_payload_assembler(header.id) return assembler.get_payload_as_stream() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py index 39e1c25e6..df6c78655 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py @@ -19,9 +19,11 @@ def read_body_as_str(self) -> str: content_stream = self.streams[0] if self.streams else None if not content_stream: + # TODO: maybe raise an error return "" # TODO: encoding double check - return bytes(content_stream.stream).decode("utf8") + stream = content_stream.stream.read_until_end() + return bytes(stream).decode("utf-8-sig") except Exception as error: raise error diff --git a/libraries/botframework-connector/botframework/connector/__init__.py b/libraries/botframework-connector/botframework/connector/__init__.py index a90857529..c9b0302b3 100644 --- a/libraries/botframework-connector/botframework/connector/__init__.py +++ b/libraries/botframework-connector/botframework/connector/__init__.py @@ -18,6 +18,12 @@ from .aiohttp_bf_pipeline import AsyncBfPipeline from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration -__all__ = ["AsyncBfPipeline", "Channels", "ConnectorClient", "EmulatorApiClient"] +__all__ = [ + "AsyncBfPipeline", + "Channels", + "ConnectorClient", + "EmulatorApiClient", + "BotFrameworkConnectorConfiguration", +] __version__ = VERSION diff --git a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py index 4d2db8aea..4f4cffe27 100644 --- a/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py +++ b/libraries/botframework-connector/botframework/connector/aio/_connector_client_async.py @@ -66,16 +66,19 @@ def __init__( *, pipeline_type: Optional[Type[AsyncPipeline]] = None, sender: Optional[AsyncHTTPSender] = None, - driver: Optional[AsyncHttpDriver] = None + driver: Optional[AsyncHttpDriver] = None, + custom_configuration: [BotFrameworkConnectorConfiguration] = None, ): - - self.config = BotFrameworkConnectorConfiguration( - credentials, - base_url, - pipeline_type=pipeline_type, - sender=sender, - driver=driver, - ) + if custom_configuration: + self.config = custom_configuration + else: + self.config = BotFrameworkConnectorConfiguration( + credentials, + base_url, + pipeline_type=pipeline_type, + sender=sender, + driver=driver, + ) super(ConnectorClient, self).__init__(self.config) client_models = { diff --git a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py index 26e46f061..2011abdf4 100644 --- a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py +++ b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py @@ -4,9 +4,11 @@ import asyncio from msrest.pipeline import AsyncPipeline, AsyncHTTPPolicy, SansIOHTTPPolicy -from msrest.pipeline.aiohttp import AioHTTPSender -from msrest.universal_http.aiohttp import AioHTTPSender as Driver -from msrest.pipeline.async_requests import AsyncRequestsCredentialsPolicy +from msrest.universal_http.async_requests import AsyncRequestsHTTPSender as Driver +from msrest.pipeline.async_requests import ( + AsyncRequestsCredentialsPolicy, + AsyncPipelineRequestsHTTPSender, +) from msrest.pipeline.universal import RawDeserializer from .bot_framework_sdk_client_async import BotFrameworkConnectorConfiguration @@ -28,13 +30,7 @@ def __init__(self, config: BotFrameworkConnectorConfiguration): # Assume this is the old credentials class, and then requests. Wrap it. policies.insert(1, AsyncRequestsCredentialsPolicy(creds)) - sender = config.sender or AioHTTPSender(config.driver or BFAioHTTPDriver) + sender = config.sender or AsyncPipelineRequestsHTTPSender( + config.driver or Driver(config) + ) super().__init__(policies, sender) - - -class BFAioHTTPDriver(Driver): - """AioHttp HTTP sender implementation. - """ - - def __del__(self): - asyncio.create_task(self._session.close()) diff --git a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py index c24bb962c..bb0fbe16b 100644 --- a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py +++ b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py @@ -28,18 +28,13 @@ def __init__( self.sender = sender self.driver = driver - if pipeline_type: - self.pipeline = pipeline_type(self) + self.custom_pipeline = pipeline_type(self) if pipeline_type else None class BotFrameworkSDKClientAsync(SDKClientAsync): def __init__(self, config: BotFrameworkConnectorConfiguration) -> None: super().__init__(config) - self._client = BotFrameworkServiceClientAsync(config) - -class BotFrameworkServiceClientAsync(ServiceClientAsync): - def __init__(self, config: BotFrameworkConnectorConfiguration) -> None: - super(ServiceClientAsync, self).__init__(config) - - self.config.pipeline = config.pipeline or self._create_default_pipeline() + self._client.config.pipeline = ( + config.custom_pipeline or self._client.config.pipeline + ) From 28f09e4e2a8592dc441228999a1eb8548a87b399 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 13 May 2020 18:22:24 -0700 Subject: [PATCH 14/37] Disassembler fixes, sender struggling to send through socket --- .../core/streaming/aiohttp_web_socket.py | 21 ++++++++----- .../core/streaming/streaming_http_client.py | 2 +- .../streaming/streaming_request_handler.py | 5 +++- .../disassemblers/payload_disassembler.py | 30 ++++++++++++------- .../transport/web_socket/web_socket_server.py | 2 +- .../web_socket/web_socket_transport.py | 2 ++ 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py index e96d22df1..174e4174e 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py @@ -51,14 +51,19 @@ async def receive(self) -> WebSocketMessage: async def send( self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool ): - if message_type == WebSocketMessageType.BINARY: - await self._aiohttp_ws.send_bytes(buffer) - elif message_type == WebSocketMessageType.TEXT: - await self._aiohttp_ws.send_str(buffer) - else: - raise RuntimeError( - f"AiohttpWebSocket - message_type: {message_type} currently not supported" - ) + try: + if message_type == WebSocketMessageType.BINARY: + # TODO: The clening buffer line should be removed, just for bypassing bug in POC + clean_buffer = bytes([byte for byte in buffer if byte is not None]) + await self._aiohttp_ws.send_bytes(clean_buffer) + elif message_type == WebSocketMessageType.TEXT: + await self._aiohttp_ws.send_str(buffer) + else: + raise RuntimeError( + f"AiohttpWebSocket - message_type: {message_type} currently not supported" + ) + except Exception as error: + raise error @property def status(self) -> WebSocketState: diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index 5b2f91692..3cb9927c8 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -67,7 +67,7 @@ def __init__( async def send(self, request: ClientRequest, **config: Any) -> AsyncClientResponse: # TODO: validate form of request to perform operations streaming_request = StreamingRequest( - path=request.url[request.url.index("/v3") :], verb=request.method + path=request.url[request.url.index("v3/") :], verb=request.method ) streaming_request.set_body(request.data) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index a6db1007f..15f084fb5 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import platform +import traceback from http import HTTPStatus from datetime import datetime from logging import Logger @@ -199,7 +200,9 @@ async def send_streaming_request( if server_response.status_code == HTTPStatus.OK: return server_response.read_body_as_json(ReceiveResponse) - except Exception: + except Exception as error: + # TODO: remove printing + traceback.print_exc() # TODO: log error pass diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py index 25e316e9a..de8098398 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import json from asyncio import Future from abc import ABC, abstractmethod from uuid import UUID @@ -43,18 +44,25 @@ async def disassemble(self): def get_stream_description(stream: ResponseMessageStream) -> StreamDescription: description = StreamDescription(id=str(int(stream.id))) - # TODO: validate statement below, also make the string a constant - content_type: List[str] = stream.content.headers().get("Content-Type") - if content_type: - description.content_type = content_type[0] + # TODO: This content type is hardcoded for POC, investigate how to proceed + content = bytes(stream.content).decode('utf8') + + try: + json.loads(content) + content_type = "application/json" + except ValueError: + content_type = "text/plain" + + description.content_type = content_type + description.length = len(content) # TODO: validate statement below, also make the string a constant - content_length: int = stream.content.headers.get("Content-Length") - if content_length: - description.length = int(content_length) - else: - # TODO: check statement validity - description.length = stream.content.headers.content_length + # content_length: int = stream.content.headers.get("Content-Length") + # if content_length: + # description.length = int(content_length) + # else: + # # TODO: check statement validity + # description.length = stream.content.headers.content_length return description @@ -86,7 +94,7 @@ async def _send(self): ) is_length_known = True - self.sender.send_payload(header, self._stream, is_length_known, self._on_sent) + self.sender.send_payload(header, self._stream, is_length_known, self._on_send) async def _on_send(self, header: Header): self._send_offset += header.payload_length diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index 112af160b..0a9cb12b1 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -66,7 +66,7 @@ async def send(self, request: StreamingRequest) -> ReceiveResponse: if not self._sender.is_connected or not self._sender.is_connected: raise RuntimeError("The server is not connected") - return await self._protocol_adapter.send_request() + return await self._protocol_adapter.send_request(request) def disconnect(self): self._sender.disconnect() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index 64f7e80ab..214b94d15 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import traceback from typing import List from botbuilder.streaming.transport import TransportReceiverBase, TransportSenderBase @@ -85,6 +86,7 @@ async def send( # Exceptions of the three types below will also have set the socket's state to closed, which fires an # event consumers of this class are subscribed to and have handling around. Any other exception needs to # be thrown to cause a non-transport-connectivity failure. + traceback.print_exc() raise error return 0 From 14320973837418bf92340ffa80d023e0bd70e4ec Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 26 May 2020 09:51:13 -0700 Subject: [PATCH 15/37] changes on disassembler and receiver --- .../streaming/payload_transport/payload_receiver.py | 4 ++++ .../streaming/payloads/disassemblers/payload_disassembler.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index c932f89b9..1d6f07c23 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import traceback + from typing import Callable, List import botbuilder.streaming as streaming @@ -83,6 +85,7 @@ async def _receive_packets(self): try: # read the header header_offset = 0 + # TODO: this while is probalby not necessary while header_offset < TransportConstants.MAX_HEADER_LENGTH: length = await self._receiver.receive( self._receive_header_buffer, @@ -145,6 +148,7 @@ async def _receive_packets(self): self._receive_action(header, content_stream, offset) except Exception as exception: + traceback.print_exc() is_closed = True disconnect_args = DisconnectedEventArgs(reason=str(exception)) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py index de8098398..18d5c6ee7 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py @@ -42,7 +42,7 @@ async def disassemble(self): @staticmethod def get_stream_description(stream: ResponseMessageStream) -> StreamDescription: - description = StreamDescription(id=str(int(stream.id))) + description = StreamDescription(id=str(stream.id)) # TODO: This content type is hardcoded for POC, investigate how to proceed content = bytes(stream.content).decode('utf8') From b29c586128ae570a586b52c5a58cb684770595f1 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Fri, 5 Mar 2021 14:02:22 -0800 Subject: [PATCH 16/37] adding streaming to ci pipeline --- .../botbuilder-core/botbuilder/core/bot_framework_adapter.py | 4 +++- .../streaming/payloads/disassemblers/payload_disassembler.py | 2 +- pipelines/botbuilder-python-ci.yml | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py index 4c4cdd63c..d973f1100 100644 --- a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py +++ b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py @@ -1271,7 +1271,9 @@ async def exchange_token_from_credentials( return result raise TypeError(f"exchange_async returned improper result: {type(result)}") - def can_process_outgoing_activity(self, activity: Activity) -> bool: + def can_process_outgoing_activity( + self, activity: Activity # pylint: unused-argument + ) -> bool: return False async def process_outgoing_activity( diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py index 18d5c6ee7..67f440d84 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py @@ -45,7 +45,7 @@ def get_stream_description(stream: ResponseMessageStream) -> StreamDescription: description = StreamDescription(id=str(stream.id)) # TODO: This content type is hardcoded for POC, investigate how to proceed - content = bytes(stream.content).decode('utf8') + content = bytes(stream.content).decode("utf8") try: json.loads(content) diff --git a/pipelines/botbuilder-python-ci.yml b/pipelines/botbuilder-python-ci.yml index 89987ffc9..b61dd1003 100644 --- a/pipelines/botbuilder-python-ci.yml +++ b/pipelines/botbuilder-python-ci.yml @@ -54,6 +54,7 @@ jobs: pip install -e ./libraries/botbuilder-integration-applicationinsights-aiohttp pip install -e ./libraries/botbuilder-adapters-slack pip install -e ./libraries/botbuilder-integration-aiohttp + pip install -e ./libraries/botbuilder-streaming pip install -r ./libraries/botframework-connector/tests/requirements.txt pip install -r ./libraries/botbuilder-core/tests/requirements.txt pip install -r ./libraries/botbuilder-ai/tests/requirements.txt From 621c79b36c46cce6c8f457b628aed5ec8a9902ea Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 10 Mar 2021 14:58:33 -0800 Subject: [PATCH 17/37] Pylint fixes --- .pylintrc | 4 +- .../botbuilder/core/bot_framework_adapter.py | 4 +- ...tp_channel_service_exception_middleware.py | 9 +---- .../bot_framework_http_adapter_base.py | 37 ++++++++++--------- .../core/streaming/streaming_http_client.py | 6 +-- .../streaming/streaming_request_handler.py | 15 +++++--- .../aiohttp/bot_framework_http_adapter.py | 6 ++- .../payload_transport/payload_receiver.py | 3 +- .../payload_transport/payload_sender.py | 3 +- .../streaming/payload_transport/send_queue.py | 5 ++- .../assemblers/payload_stream_assembler.py | 2 +- .../assemblers/receive_request_assembler.py | 1 + .../assemblers/receive_response_assembler.py | 1 + .../disassemblers/request_disassembler.py | 1 - .../disassemblers/response_disassembler.py | 1 - .../streaming/payloads/header_serializer.py | 20 ++++++---- .../streaming/payloads/models/header.py | 1 + .../payloads/models/stream_description.py | 1 + .../payloads/payload_assembler_manager.py | 5 +-- .../payloads/response_message_stream.py | 1 + .../streaming/payloads/send_operations.py | 1 - .../streaming/payloads/stream_manager.py | 1 - .../botbuilder/streaming/receive_response.py | 2 +- .../transport/web_socket/web_socket_server.py | 5 ++- .../web_socket/web_socket_transport.py | 3 +- .../connector/aiohttp_bf_pipeline.py | 2 - .../connector/auth/emulator_validation.py | 1 - .../bot_framework_sdk_client_async.py | 2 +- 28 files changed, 78 insertions(+), 65 deletions(-) diff --git a/.pylintrc b/.pylintrc index a134068ff..d0ec3b74a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -158,7 +158,9 @@ disable=print-statement, too-many-return-statements, import-error, no-name-in-module, - too-many-branches + too-many-branches, + too-many-ancestors, + too-many-nested-blocks # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py index d973f1100..751ce3e8c 100644 --- a/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py +++ b/libraries/botbuilder-core/botbuilder/core/bot_framework_adapter.py @@ -1272,14 +1272,14 @@ async def exchange_token_from_credentials( raise TypeError(f"exchange_async returned improper result: {type(result)}") def can_process_outgoing_activity( - self, activity: Activity # pylint: unused-argument + self, activity: Activity # pylint: disable=unused-argument ) -> bool: return False async def process_outgoing_activity( self, turn_context: TurnContext, activity: Activity ) -> ResourceResponse: - raise NotImplementedError() + raise Exception("NotImplemented") @staticmethod def key_for_connector_client(service_url: str, app_id: str, scope: str): diff --git a/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py b/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py index fb9d9d3cd..d58073d06 100644 --- a/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py +++ b/libraries/botbuilder-core/botbuilder/core/integration/aiohttp_channel_service_exception_middleware.py @@ -27,11 +27,6 @@ async def aiohttp_error_middleware(request, handler): raise HTTPUnauthorized() except KeyError: raise HTTPNotFound() - except Exception as error: - try: - raise error - raise HTTPInternalServerError() - except: - pass - + except Exception: traceback.print_exc() + raise HTTPInternalServerError() diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index ac0b55954..4e84c9632 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -28,6 +28,7 @@ class BotFrameworkHttpAdapterBase(BotFrameworkAdapter, StreamingActivityProcessor): + # pylint: disable=pointless-string-statement def __init__(self, settings: BotFrameworkAdapterSettings): super().__init__(settings) @@ -104,29 +105,29 @@ async def send_streaming_activity(self, activity: Activity) -> ResourceResponse: return await correct_handler.send_activity(activity) return await possible_handlers[0].send_activity(activity) - else: - if self.connected_bot: - # This is a proactive message that will need a new streaming connection opened. - # The ServiceUrl of a streaming connection follows the pattern "url:[ChannelName]:[Protocol]:[Host]". - uri = activity.service_url.split(":") - protocol = uri[len(uri) - 2] - host = uri[len(uri) - 1] - # TODO: discuss if should abstract this from current package - # TODO: manage life cycle of sessions (when should we close them) - session = ClientSession() - aiohttp_ws = await session.ws_connect(protocol + host + "/api/messages") - web_socket = AiohttpWebSocket(aiohttp_ws, session) - handler = StreamingRequestHandler(self.connected_bot, self, web_socket) + if self.connected_bot: + # This is a proactive message that will need a new streaming connection opened. + # The ServiceUrl of a streaming connection follows the pattern "url:[ChannelName]:[Protocol]:[Host]". - if self.request_handlers is None: - self.request_handlers = [] + uri = activity.service_url.split(":") + protocol = uri[len(uri) - 2] + host = uri[len(uri) - 1] + # TODO: discuss if should abstract this from current package + # TODO: manage life cycle of sessions (when should we close them) + session = ClientSession() + aiohttp_ws = await session.ws_connect(protocol + host + "/api/messages") + web_socket = AiohttpWebSocket(aiohttp_ws, session) + handler = StreamingRequestHandler(self.connected_bot, self, web_socket) - self.request_handlers.append(handler) + if self.request_handlers is None: + self.request_handlers = [] - return await handler.send_activity(activity) + self.request_handlers.append(handler) - return None + return await handler.send_activity(activity) + + return None def can_process_outgoing_activity(self, activity: Activity) -> bool: if not activity: diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index 3cb9927c8..2277ae384 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import asyncio from http import HTTPStatus from logging import Logger from typing import Any @@ -64,7 +63,9 @@ def __init__( self._request_handler = request_handler self._logger = logger - async def send(self, request: ClientRequest, **config: Any) -> AsyncClientResponse: + async def send( + self, request: ClientRequest, **config: Any # pylint: disable=unused-argument + ) -> AsyncClientResponse: # TODO: validate form of request to perform operations streaming_request = StreamingRequest( path=request.url[request.url.index("v3/") :], verb=request.method @@ -91,6 +92,5 @@ async def _send_request( except Exception as error: # TODO: log error raise error - pass return None diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index 15f084fb5..fe2cdb1a0 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -84,11 +84,12 @@ def forget_conversation(self, conversation_id: str): async def process_request( self, request: ReceiveRequest, logger: Logger, context: object ) -> StreamingResponse: + # pylint: disable=pointless-string-statement response = StreamingResponse() # We accept all POSTs regardless of path, but anything else requires special treatment. if not request.verb == StreamingRequest.POST: - return self._handle_custom_paths() + return self._handle_custom_paths(request, response) # Convert the StreamingRequest into an activity the adapter can understand. try: @@ -200,11 +201,9 @@ async def send_streaming_request( if server_response.status_code == HTTPStatus.OK: return server_response.read_body_as_json(ReceiveResponse) - except Exception as error: - # TODO: remove printing + except Exception: + # TODO: remove printing and log it traceback.print_exc() - # TODO: log error - pass return None @@ -252,7 +251,11 @@ def validate_int_list(obj: object) -> bool: return None - def _server_disconnected(self, sender: object, event: DisconnectedEventArgs): + def _server_disconnected( + self, + sender: object, # pylint: disable=unused-argument + event: DisconnectedEventArgs, # pylint: disable=unused-argument + ): self._server_is_connected = False def _handle_custom_paths( diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py index 0ce0905c8..09e1507e1 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -12,7 +12,7 @@ HTTPUnauthorized, HTTPUnsupportedMediaType, ) -from botbuilder.core import Bot, BotFrameworkAdapterSettings, BotFrameworkAdapter +from botbuilder.core import Bot, BotFrameworkAdapterSettings from botbuilder.core.streaming import ( AiohttpWebSocket, BotFrameworkHttpAdapterBase, @@ -24,6 +24,7 @@ class BotFrameworkHttpAdapter(BotFrameworkHttpAdapterBase): def __init__(self, settings: BotFrameworkAdapterSettings): + # pylint: disable=invalid-name super().__init__(settings) self._AUTH_HEADER_NAME = "authorization" @@ -97,12 +98,13 @@ async def _connect_web_socket( await request_handler.listen() except Exception as error: - import traceback + import traceback # pylint: disable=import-outside-toplevel traceback.print_exc() raise Exception(f"Unable to create transport server. Error: {str(error)}") async def _http_authenticate_request(self, request: Request) -> bool: + # pylint: disable=no-member try: if not await self._credential_provider.is_authentication_disabled(): auth_header = request.headers.get(self._AUTH_HEADER_NAME) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index 1d6f07c23..f88432359 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -70,7 +70,8 @@ def disconnect(self, event_args: DisconnectedEventArgs = None): self._receiver = None if did_disconnect: - if self.disconnected: + if callable(self.disconnected): + # pylint: disable=not-callable self.disconnected( self, event_args or DisconnectedEventArgs.empty ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index 69f31c7d3..711378a8a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -80,7 +80,8 @@ def disconnect(self, event_args: DisconnectedEventArgs = None): if did_disconnect: self._connected_event.clear() - if self.disconnected: + if callable(self.disconnected): + # pylint: disable=not-callable self.disconnected( self, event_args or DisconnectedEventArgs.empty ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index 7c5bde2b2..a64d64dc3 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -1,11 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import traceback +# import traceback from queue import Queue from typing import Awaitable, Callable -from threading import Event, Lock, Semaphore + +# from threading import Event, Lock, Semaphore import asyncio import threading diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index 434b02770..a600d7fee 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -13,6 +13,7 @@ class PayloadStreamAssembler(Assembler): + # pylint: disable=super-init-not-called def __init__( self, stream_manager: "payloads.StreamManager", @@ -20,7 +21,6 @@ def __init__( type: str = None, length: int = None, ): - self._stream_manager = stream_manager or payloads.StreamManager() self._stream: "streaming.PayloadStream" = None self._lock = Lock() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py index 926e99f46..b465ed8e0 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -14,6 +14,7 @@ class ReceiveRequestAssembler(Assembler): + # pylint: disable=super-init-not-called def __init__( self, header: Header, diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py index 75363eb8a..b7af0ee1c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py @@ -14,6 +14,7 @@ class ReceiveResponseAssembler(Assembler): + # pylint: disable=super-init-not-called def __init__( self, header: Header, diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py index 065a29062..2cbccc211 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py @@ -4,7 +4,6 @@ from uuid import UUID from typing import List -import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.models import PayloadTypes, RequestPayload diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py index 6548c3f57..038b4c0b8 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py @@ -4,7 +4,6 @@ from uuid import UUID from typing import List -import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.models import PayloadTypes, ResponsePayload diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py index 7ed704be4..089293d8b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py @@ -8,15 +8,15 @@ from .models import Header -_char_to_binary_int = {val.decode(): list(val)[0] for val in [b".", b"\n", b"1", b"0"]} +_CHAR_TO_BINARY_INT = {val.decode(): list(val)[0] for val in [b".", b"\n", b"1", b"0"]} # TODO: consider abstracting the binary int list logic into a class for easier handling class HeaderSerializer: - DELIMITER = _char_to_binary_int["."] - TERMINATOR = _char_to_binary_int["\n"] - END = _char_to_binary_int["1"] - NOT_END = _char_to_binary_int["0"] + DELIMITER = _CHAR_TO_BINARY_INT["."] + TERMINATOR = _CHAR_TO_BINARY_INT["\n"] + END = _CHAR_TO_BINARY_INT["1"] + NOT_END = _CHAR_TO_BINARY_INT["0"] TYPE_OFFSET = 0 TYPE_DELIMITER_OFFSET = 1 LENGTH_OFFSET = 2 @@ -29,7 +29,11 @@ class HeaderSerializer: TERMINATOR_OFFSET = 47 @staticmethod - def serialize(header: Header, buffer: List[int], offset: int) -> int: + def serialize( + header: Header, + buffer: List[int], + offset: int, # pylint: disable=unused-argument + ) -> int: # write type buffer[HeaderSerializer.TYPE_OFFSET] = HeaderSerializer._char_to_binary_int( @@ -66,7 +70,9 @@ def serialize(header: Header, buffer: List[int], offset: int) -> int: return TransportConstants.MAX_HEADER_LENGTH @staticmethod - def deserialize(buffer: List[int], offset: int, count: int) -> Header: + def deserialize( + buffer: List[int], offset: int, count: int # pylint: disable=unused-argument + ) -> Header: if count != TransportConstants.MAX_HEADER_LENGTH: raise ValueError("Cannot deserialize header, incorrect length") diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py index c07d4d4bb..135749309 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py @@ -7,6 +7,7 @@ class Header: + # pylint: disable=invalid-name def __init__(self, *, type: str = None, id: UUID = None, end: bool = None): self._internal_payload_length = None self.type: str = type diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py index 154520f79..c426f5de1 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py @@ -7,6 +7,7 @@ class StreamDescription(Serializable): + # pylint: disable=invalid-name def __init__(self, *, id: str = None, content_type: str = None, length: int = None): self.id = id self.content_type = content_type diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py index 81d5fe07f..823efa536 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py @@ -4,7 +4,6 @@ from uuid import UUID from typing import Awaitable, Callable, Dict, List, Union -import botbuilder.streaming as streaming from botbuilder.streaming.payloads.assemblers import ( Assembler, ReceiveRequestAssembler, @@ -33,7 +32,7 @@ def get_payload_stream( # TODO: The return value SHOULDN'T be a union, we should interface List[int] into a BFStream class if self._is_stream_payload(header): return self._stream_manager.get_payload_stream(header) - elif not self._active_assemblers.get(header.id): + if not self._active_assemblers.get(header.id): # a new requestId has come in, start a new task to process it as it is received assembler = self._create_payload_assembler(header) if assembler: @@ -63,7 +62,7 @@ def _create_payload_assembler(self, header: Header) -> Assembler: return ReceiveRequestAssembler( header, self._stream_manager, self._on_receive_request ) - elif header.type == PayloadTypes.RESPONSE: + if header.type == PayloadTypes.RESPONSE: return ReceiveResponseAssembler( header, self._stream_manager, self._on_receive_response ) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py index 716f604b7..04ae1dd77 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py @@ -5,6 +5,7 @@ class ResponseMessageStream: + # pylint: disable=invalid-name def __init__(self, *, id: UUID = None, content: object = None): self.id = id or uuid4() self.content = content diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py index c8ed122d9..2e81e255d 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py @@ -4,7 +4,6 @@ import asyncio from uuid import UUID -import botbuilder.streaming as streaming from botbuilder.streaming.payload_transport import PayloadSender from botbuilder.streaming.payloads.disassemblers import ( CancelDisassembler, diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py index a9b4a0d83..38bf96f9f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py @@ -4,7 +4,6 @@ from uuid import UUID from typing import Callable, Dict, List -import botbuilder.streaming as streaming from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler from botbuilder.streaming.payloads.models import Header diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index f7680b464..23d7a25f4 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -35,7 +35,7 @@ def read_body_as_str(self) -> str: return "" # TODO: encoding double check - return bytes(content_stream.stream).decode("utf8") + return content_stream.decode("utf8") except Exception as error: raise error diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index 0a9cb12b1..3582a25f5 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -72,7 +72,9 @@ def disconnect(self): self._sender.disconnect() self._receiver.disconnect() - def _on_connection_disconnected(self, sender: object, event_args: object): + def _on_connection_disconnected( + self, sender: object, event_args: object # pylint: disable=unused-argument + ): if not self._is_disconnecting: self._is_disconnecting = True @@ -84,6 +86,7 @@ def _on_connection_disconnected(self, sender: object, event_args: object): sender.disconnect() if self.disconnected_event_handler: + # pylint: disable=not-callable self.disconnected_event_handler(self, DisconnectedEventArgs.empty) self._is_disconnecting = False diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index 214b94d15..b8070d0be 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -31,13 +31,14 @@ async def close(self): "Closed by the WebSocketTransport", ) except Exception: + # pylint: disable=pointless-string-statement """ Any exception thrown here will be caused by the socket already being closed, which is the state we want to put it in by calling this method, which means we don't care if it was already closed and threw an exception when we tried to close it again. """ - pass + traceback.print_exc() # TODO: might need to remove offset and count if no segmentation possible # TODO: considering to create a BFTransportBuffer class to abstract the logic of binary buffers adapting to diff --git a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py index 2011abdf4..b46a40857 100644 --- a/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py +++ b/libraries/botframework-connector/botframework/connector/aiohttp_bf_pipeline.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import asyncio - from msrest.pipeline import AsyncPipeline, AsyncHTTPPolicy, SansIOHTTPPolicy from msrest.universal_http.async_requests import AsyncRequestsHTTPSender as Driver from msrest.pipeline.async_requests import ( diff --git a/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py b/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py index 64e794489..a50a5eaea 100644 --- a/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py +++ b/libraries/botframework-connector/botframework/connector/auth/emulator_validation.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import asyncio from typing import Union import jwt diff --git a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py index bb0fbe16b..9efb15b7d 100644 --- a/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py +++ b/libraries/botframework-connector/botframework/connector/bot_framework_sdk_client_async.py @@ -3,7 +3,7 @@ from typing import Optional, Type -from msrest.async_client import SDKClientAsync, ServiceClientAsync +from msrest.async_client import SDKClientAsync from msrest.universal_http.async_abc import AsyncHTTPSender as AsyncHttpDriver from msrest.pipeline import AsyncPipeline from msrest.pipeline.aiohttp import AsyncHTTPSender From aaa806813fa5472e62c4bb25ecdcc83f1ef64ecc Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 10 Mar 2021 16:24:56 -0800 Subject: [PATCH 18/37] updated streaming setup.py --- libraries/botbuilder-streaming/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botbuilder-streaming/setup.py index 05eb2f679..6f3c2c4cb 100644 --- a/libraries/botbuilder-streaming/setup.py +++ b/libraries/botbuilder-streaming/setup.py @@ -4,11 +4,11 @@ import os from setuptools import setup -VERSION = os.environ["packageVersion"] if "packageVersion" in os.environ else "4.7.1" +VERSION = os.environ["packageVersion"] if "packageVersion" in os.environ else "4.12.0" REQUIRES = [ - "botbuilder-schema>=4.7.1", - "botframework-connector>=4.7.1", - "botbuilder-core>=4.7.1", + "botbuilder-schema>=4.12.0", + "botframework-connector>=4.12.0", + "botbuilder-core>=4.12.0", ] root = os.path.abspath(os.path.dirname(__file__)) From a0424c2d9e41571656950ebfdfd050bae36a778f Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 10 Mar 2021 16:50:39 -0800 Subject: [PATCH 19/37] Removing 3.6 --- pipelines/botbuilder-python-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pipelines/botbuilder-python-ci.yml b/pipelines/botbuilder-python-ci.yml index b61dd1003..221f1a1c3 100644 --- a/pipelines/botbuilder-python-ci.yml +++ b/pipelines/botbuilder-python-ci.yml @@ -6,7 +6,6 @@ variables: COVERALLS_GIT_COMMIT: $(Build.SourceVersion) COVERALLS_SERVICE_JOB_ID: $(Build.BuildId) COVERALLS_SERVICE_NAME: python-ci - python.36: 3.6.x python.37: 3.7.x python.38: 3.8.x # PythonCoverallsToken: get this from Azure @@ -20,8 +19,6 @@ jobs: strategy: matrix: - Python36: - PYTHON_VERSION: '$(python.36)' Python37: PYTHON_VERSION: '$(python.37)' Python38: @@ -38,9 +35,6 @@ jobs: inputs: versionSpec: '$(PYTHON_VERSION)' - - script: 'sudo ln -s /opt/hostedtoolcache/Python/3.6.9/x64/lib/libpython3.6m.so.1.0 /usr/lib/libpython3.6m.so' - displayName: libpython3.6m - - script: | python -m pip install --upgrade pip pip install -e ./libraries/botbuilder-schema From b1795241c29481852ca346f35ed3c1a92796301c Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 24 Mar 2021 16:46:54 -0700 Subject: [PATCH 20/37] Changing all concurrent mechanisms in streaming to asyncio --- .../streaming/streaming_request_handler.py | 6 +++-- .../botbuilder/streaming/payload_stream.py | 14 +++++----- .../payload_transport/payload_sender.py | 7 +++-- .../streaming/payload_transport/send_queue.py | 26 +++++-------------- .../assemblers/payload_stream_assembler.py | 8 +++--- .../assemblers/receive_request_assembler.py | 17 +++--------- .../assemblers/receive_response_assembler.py | 16 +++--------- .../botbuilder/streaming/receive_request.py | 4 +-- 8 files changed, 35 insertions(+), 63 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index fe2cdb1a0..4e10f1313 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -93,8 +93,9 @@ async def process_request( # Convert the StreamingRequest into an activity the adapter can understand. try: - body_str = request.read_body_as_str() + body_str = await request.read_body_as_str() except Exception as error: + traceback.print_exc() response.status_code = int(HTTPStatus.BAD_REQUEST) # TODO: log error @@ -149,6 +150,7 @@ async def process_request( response.set_body(adapter_response.body) except Exception as error: + traceback.print_exc() response.status_code = int(HTTPStatus.INTERNAL_SERVER_ERROR) response.set_body(str(error)) # TODO: log error @@ -184,7 +186,7 @@ async def send_activity(self, activity: Activity) -> ResourceResponse: return server_response.read_body_as_json(ResourceResponse) except Exception: # TODO: log error - pass + traceback.print_exc() return None diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py index 313957787..c92957cd1 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from threading import Lock, Semaphore +from asyncio import Lock, Semaphore from typing import List from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler @@ -34,13 +34,13 @@ def write(self, buffer: List[int], offset: int, count: int): buffer_copy = buffer[offset : offset + count] self.give_buffer(buffer_copy) - def read(self, buffer: List[int], offset: int, count: int): + async def read(self, buffer: List[int], offset: int, count: int): if self._end: return 0 if not self._active: - self._data_available.acquire() - with self._lock: + await self._data_available.acquire() + async with self._lock: self._active.extend(self._buffer) self._buffer = [] @@ -64,12 +64,14 @@ def read(self, buffer: List[int], offset: int, count: int): return available_count - def read_until_end(self): + async def read_until_end(self): result = [None] * self._assembler.content_length current_size = 0 while not self._end: - count = self.read(result, current_size, self._assembler.content_length) + count = await self.read( + result, current_size, self._assembler.content_length + ) current_size += count return result diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index 711378a8a..f408c522e 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -1,8 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import asyncio -from threading import Event +from asyncio import Event, ensure_future from typing import Awaitable, Callable, List from botbuilder.streaming.transport import ( @@ -89,7 +88,7 @@ def disconnect(self, event_args: DisconnectedEventArgs = None): self._is_disconnecting = False async def _write_packet(self, packet: SendPacket): - self._connected_event.wait() + await self._connected_event.wait() try: # determine if we know the payload length and end @@ -144,7 +143,7 @@ async def _write_packet(self, packet: SendPacket): if packet.sent_callback: # TODO: should this really run in the background? - asyncio.create_task(packet.sent_callback(packet.header)) + ensure_future(packet.sent_callback(packet.header)) except Exception as exception: disconnected_args = DisconnectedEventArgs(reason=str(exception)) self.disconnect(disconnected_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index a64d64dc3..df8f32909 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -1,16 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# import traceback +import traceback -from queue import Queue +from asyncio import Queue, ensure_future, sleep from typing import Awaitable, Callable -# from threading import Event, Lock, Semaphore - -import asyncio -import threading - class SendQueue: def __init__(self, action: Callable[[object], Awaitable], timeout: int = 30): @@ -20,26 +15,20 @@ def __init__(self, action: Callable[[object], Awaitable], timeout: int = 30): self._timeout_seconds = timeout # TODO: this have to be abstracted so can remove asyncio dependency - def schedule_task(): - loop = asyncio.new_event_loop() - loop.run_until_complete(self._process()) - - new_thread = threading.Thread(target=schedule_task, args=()) - new_thread.daemon = True - new_thread.start() + ensure_future(self._process()) def post(self, item: object): self._post_internal(item) def _post_internal(self, item: object): - self._queue.put(item) + self._queue.put_nowait(item) async def _process(self): while True: try: while True: - await asyncio.sleep(1) - item = self._queue.get(block=False) + await sleep(1) + item = await self._queue.get() try: await self._action(item) except Exception: @@ -49,5 +38,4 @@ async def _process(self): self._queue.task_done() except Exception: # AppInsights.TrackException(e) - # traceback.print_exc() - pass + traceback.print_exc() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index a600d7fee..37fa1d109 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -3,7 +3,6 @@ from uuid import UUID from typing import List -from threading import Lock import botbuilder.streaming as streaming import botbuilder.streaming.payloads as payloads @@ -23,7 +22,7 @@ def __init__( ): self._stream_manager = stream_manager or payloads.StreamManager() self._stream: "streaming.PayloadStream" = None - self._lock = Lock() + # self._lock = Lock() self.identifier = identifier self.content_type = type self.content_length = length @@ -33,9 +32,8 @@ def create_stream_from_payload(self) -> "streaming.PayloadStream": return streaming.PayloadStream(self) def get_payload_as_stream(self) -> "streaming.PayloadStream": - with self._lock: - if not self._stream: - self._stream = self.create_stream_from_payload() + if not self._stream: + self._stream = self.create_stream_from_payload() return self._stream diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py index b465ed8e0..0a78acd71 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py @@ -3,7 +3,6 @@ import asyncio from uuid import UUID -from threading import Lock, Thread from typing import Awaitable, Callable, List import botbuilder.streaming as streaming @@ -32,16 +31,14 @@ def __init__( self._on_completed = on_completed self.identifier = header.id self._length = header.payload_length if header.end else None - self._lock = Lock() self._stream: List[int] = None def create_stream_from_payload(self) -> List[int]: return [None] * (self._length or 0) def get_payload_as_stream(self) -> List[int]: - with self._lock: - if self._stream is None: - self._stream = self.create_stream_from_payload() + if self._stream is None: + self._stream = self.create_stream_from_payload() return self._stream @@ -49,14 +46,8 @@ def on_receive(self, header: Header, stream: List[int], content_length: int): if header.end: self.end = True - # Execute the request on a separate Thread in the background - def schedule_task(): - loop = asyncio.new_event_loop() - loop.run_until_complete(self.process_request(stream)) - - new_thread = Thread(target=schedule_task, args=()) - new_thread.daemon = True - new_thread.start() + # Execute the request in the background + asyncio.ensure_future(self.process_request(stream)) def close(self): self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py index b7af0ee1c..f5602e508 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py @@ -3,7 +3,6 @@ import asyncio from uuid import UUID -from threading import Lock, Thread from typing import Awaitable, Callable, List import botbuilder.streaming as streaming @@ -32,16 +31,14 @@ def __init__( self._on_completed = on_completed self.identifier = header.id self._length = header.payload_length if header.end else None - self._lock = Lock() self._stream: List[int] = None def create_stream_from_payload(self) -> List[int]: return [None] * (self._length or 0) def get_payload_as_stream(self) -> List[int]: - with self._lock: - if self._stream is None: - self._stream = self.create_stream_from_payload() + if self._stream is None: + self._stream = self.create_stream_from_payload() return self._stream @@ -51,13 +48,8 @@ def on_receive(self, header: Header, stream: List[int], content_length: int): # Execute the response on a separate Task # Execute the request on a separate Thread in the background - def schedule_task(): - loop = asyncio.new_event_loop() - loop.run_until_complete(self.process_response(stream)) - - new_thread = Thread(target=schedule_task, args=()) - new_thread.daemon = True - new_thread.start() + # Execute the request on a separate in the background + asyncio.ensure_future(self.process_response(stream)) def close(self): self._stream_manager.close_stream(self.identifier) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py index df6c78655..c6316edfd 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py @@ -14,7 +14,7 @@ def __init__( self.path = path self.streams: List[ContentStream] = streams or [] - def read_body_as_str(self) -> str: + async def read_body_as_str(self) -> str: try: content_stream = self.streams[0] if self.streams else None @@ -23,7 +23,7 @@ def read_body_as_str(self) -> str: return "" # TODO: encoding double check - stream = content_stream.stream.read_until_end() + stream = await content_stream.stream.read_until_end() return bytes(stream).decode("utf-8-sig") except Exception as error: raise error From 87c41119268d32a10485f5ca8c2da1f43b3cad1b Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 30 Mar 2021 14:28:17 -0700 Subject: [PATCH 21/37] Added validation for abrupt closing of websocket, added tracebacks and async validation for disconnecting callbacks --- .../core/streaming/aiohttp_web_socket.py | 17 ++++++++++---- .../payload_transport/payload_receiver.py | 22 +++++++++++++------ .../payload_transport/payload_sender.py | 21 +++++++++++++----- .../streaming/payload_transport/send_queue.py | 3 +-- .../streaming/payloads/request_manager.py | 4 ++-- .../transport/web_socket/web_socket_server.py | 17 +++++++++----- 6 files changed, 57 insertions(+), 27 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py index 174e4174e..5e351eb70 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py @@ -2,6 +2,8 @@ # Licensed under the MIT License. import asyncio +import traceback + from typing import Any, Optional, Union from aiohttp import ClientWebSocketResponse, WSMsgType, ClientSession @@ -38,19 +40,25 @@ async def receive(self) -> WebSocketMessage: try: message = await self._aiohttp_ws.receive() + if message.type == WSMsgType.TEXT: + message_data = list(str(message.data).encode("ascii")) + elif message.type == WSMsgType.BINARY: + message_data = list(message.data) + elif isinstance(message.data, int): + message_data = [] + # async for message in self._aiohttp_ws: return WebSocketMessage( - message_type=WebSocketMessageType(int(message.type)), - data=list(str(message.data).encode("ascii")) - if message.type == WSMsgType.TEXT - else list(message.data), + message_type=WebSocketMessageType(int(message.type)), data=message_data ) except Exception as error: + traceback.print_exc() raise error async def send( self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool ): + is_closing = self._aiohttp_ws.closed try: if message_type == WebSocketMessageType.BINARY: # TODO: The clening buffer line should be removed, just for bypassing bug in POC @@ -63,6 +71,7 @@ async def send( f"AiohttpWebSocket - message_type: {message_type} currently not supported" ) except Exception as error: + traceback.print_exc() raise error @property diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index f88432359..df054ced8 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -3,6 +3,7 @@ import traceback +from asyncio import iscoroutinefunction, isfuture from typing import Callable, List import botbuilder.streaming as streaming @@ -53,7 +54,7 @@ def subscribe( self._get_stream = get_stream self._receive_action = receive_action - def disconnect(self, event_args: DisconnectedEventArgs = None): + async def disconnect(self, event_args: DisconnectedEventArgs = None): did_disconnect = False if not self._is_disconnecting: @@ -61,20 +62,27 @@ def disconnect(self, event_args: DisconnectedEventArgs = None): try: try: if self._receiver: - self._receiver.close() + await self._receiver.close() # TODO: investigate if 'dispose' is necessary did_disconnect = True except Exception: - pass + traceback.print_exc() self._receiver = None if did_disconnect: if callable(self.disconnected): # pylint: disable=not-callable - self.disconnected( - self, event_args or DisconnectedEventArgs.empty - ) + if iscoroutinefunction(self.disconnected) or isfuture( + self.disconnected + ): + await self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) + else: + self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) finally: self._is_disconnecting = False @@ -153,4 +161,4 @@ async def _receive_packets(self): is_closed = True disconnect_args = DisconnectedEventArgs(reason=str(exception)) - self.disconnect(disconnect_args) + await self.disconnect(disconnect_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index f408c522e..e8838a1c3 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from asyncio import Event, ensure_future +from asyncio import Event, ensure_future, iscoroutinefunction, isfuture from typing import Awaitable, Callable, List from botbuilder.streaming.transport import ( @@ -61,7 +61,7 @@ def send_payload( self._send_queue.post(packet) - def disconnect(self, event_args: DisconnectedEventArgs = None): + async def disconnect(self, event_args: DisconnectedEventArgs = None): did_disconnect = False if not self._is_disconnecting: @@ -81,9 +81,16 @@ def disconnect(self, event_args: DisconnectedEventArgs = None): self._connected_event.clear() if callable(self.disconnected): # pylint: disable=not-callable - self.disconnected( - self, event_args or DisconnectedEventArgs.empty - ) + if iscoroutinefunction(self.disconnected) or isfuture( + self.disconnected + ): + await self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) + else: + self.disconnected( + self, event_args or DisconnectedEventArgs.empty + ) finally: self._is_disconnecting = False @@ -141,9 +148,11 @@ async def _write_packet(self, packet: SendPacket): # TODO: make custom exception raise Exception("TransportDisconnectedException") + offset += count + if packet.sent_callback: # TODO: should this really run in the background? ensure_future(packet.sent_callback(packet.header)) except Exception as exception: disconnected_args = DisconnectedEventArgs(reason=str(exception)) - self.disconnect(disconnected_args) + await self.disconnect(disconnected_args) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index df8f32909..07d197496 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -32,8 +32,7 @@ async def _process(self): try: await self._action(item) except Exception: - # AppInsights.TrackException(e) - pass + traceback.print_exc() finally: self._queue.task_done() except Exception: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py index c3138745d..ccec9a176 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from asyncio import Future +from asyncio import Future, shield from uuid import UUID from typing import Dict @@ -38,7 +38,7 @@ async def get_response(self, request_id: UUID) -> "streaming.ReceiveResponse": self._pending_requests[request_id] = pending_request try: - response: streaming.ReceiveResponse = await pending_request + response: streaming.ReceiveResponse = await shield(pending_request) return response finally: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py index 3582a25f5..cad838343 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from asyncio import Future +from asyncio import Future, iscoroutinefunction, isfuture from typing import Callable from botbuilder.streaming import ( @@ -68,11 +68,11 @@ async def send(self, request: StreamingRequest) -> ReceiveResponse: return await self._protocol_adapter.send_request(request) - def disconnect(self): - self._sender.disconnect() - self._receiver.disconnect() + async def disconnect(self): + await self._sender.disconnect() + await self._receiver.disconnect() - def _on_connection_disconnected( + async def _on_connection_disconnected( self, sender: object, event_args: object # pylint: disable=unused-argument ): if not self._is_disconnecting: @@ -83,7 +83,12 @@ def _on_connection_disconnected( self._closed_signal = None if sender in [self._sender, self._receiver]: - sender.disconnect() + if iscoroutinefunction(sender.disconnect) or isfuture( + sender.disconnect + ): + await sender.disconnect() + else: + sender.disconnect() if self.disconnected_event_handler: # pylint: disable=not-callable From d6702372676b0b103e1ec0d4165e35c31e36b188 Mon Sep 17 00:00:00 2001 From: msomanathan Date: Mon, 19 Apr 2021 11:24:55 -0700 Subject: [PATCH 22/37] UnblockActivityProcessorThread --- .../streaming/payloads/assemblers/payload_stream_assembler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py index 37fa1d109..6f19799cd 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py @@ -32,7 +32,7 @@ def create_stream_from_payload(self) -> "streaming.PayloadStream": return streaming.PayloadStream(self) def get_payload_as_stream(self) -> "streaming.PayloadStream": - if not self._stream: + if self._stream is None: self._stream = self.create_stream_from_payload() return self._stream From a194366baab224728f402595eac778913faa81a0 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 21 Apr 2021 15:51:35 -0700 Subject: [PATCH 23/37] Header serialization fix and stream serialization fix. --- .../disassemblers/response_message_stream_disassembler.py | 8 ++++---- .../botbuilder/streaming/payloads/header_serializer.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py index a7123c46e..b25356df0 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py @@ -18,10 +18,10 @@ def __init__(self, sender: PayloadSender, content_stream: ResponseMessageStream) @property def type(self) -> str: - return PayloadTypes.REQUEST + return PayloadTypes.STREAM async def get_stream(self) -> List[int]: - # TODO: align logic below to the shape of content_stream.content - stream: List[int] = list(str(self.content_stream.content).encode()) + # TODO: check if bypass is correct here or if serialization should take place. + # this is redundant -->stream: List[int] = list(str(self.content_stream.content).encode()) - return stream + return self.content_stream.content diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py index 089293d8b..fc2348c98 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py @@ -65,7 +65,7 @@ def serialize( buffer[HeaderSerializer.END_OFFSET] = ( HeaderSerializer.END if header.end else HeaderSerializer.NOT_END ) - buffer[HeaderSerializer.TERMINATOR_OFFSET] = HeaderSerializer.DELIMITER + buffer[HeaderSerializer.TERMINATOR_OFFSET] = HeaderSerializer.TERMINATOR return TransportConstants.MAX_HEADER_LENGTH @@ -153,7 +153,7 @@ def _int_to_formatted_encoded_str(value: int, str_format: str) -> bytes: @staticmethod def _uuid_to_numeric_encoded_str(value: UUID) -> bytes: - return str(int(value)).encode("ascii") + return str(value).encode("ascii") @staticmethod def _binary_int_to_char(binary_int: int) -> str: From a56176a163bb5002270388892b30b505b34e7b1f Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Fri, 23 Apr 2021 14:37:53 -0700 Subject: [PATCH 24/37] Parity change in the internal buffer structure of the payload_stream object, fixes on stream writing behavior in web_socket_transport and send response instead of request fix in protocol adapter. --- .../botbuilder/streaming/payload_stream.py | 10 +++++----- .../botbuilder/streaming/protocol_adapter.py | 4 ++-- .../transport/web_socket/web_socket_transport.py | 15 +++++---------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py index c92957cd1..9b1535d14 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py @@ -10,7 +10,7 @@ class PayloadStream: def __init__(self, assembler: PayloadStreamAssembler): self._assembler = assembler - self._buffer: List[int] = [] + self._buffer_queue: List[List[int]] = [] self._lock = Lock() self._data_available = Semaphore(0) self._producer_length = 0 # total length @@ -20,10 +20,11 @@ def __init__(self, assembler: PayloadStreamAssembler): self._end = False def __len__(self): - return len(self._buffer) + return _producer_length def give_buffer(self, buffer: List[int]): - self._buffer.extend(buffer) + self._buffer_queue.append(buffer) + self._producer_length += len(buffer) self._data_available.release() @@ -41,8 +42,7 @@ async def read(self, buffer: List[int], offset: int, count: int): if not self._active: await self._data_available.acquire() async with self._lock: - self._active.extend(self._buffer) - self._buffer = [] + self._active = self._buffer_queue.pop(0) available_count = min(len(self._active) - self._active_offset, count) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py index b1aeabeb6..59008e32e 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py @@ -55,9 +55,9 @@ async def send_request(self, request: StreamingRequest) -> ReceiveResponse: response_task = self._request_manager.get_response(request_id) request_task = self._send_operations.send_request(request_id, request) - [request, _] = await asyncio.gather(request_task, response_task) + [_, response] = await asyncio.gather(request_task, response_task) - return request + return response async def _on_receive_request(self, identifier: UUID, request: ReceiveRequest): # request is done, we can handle it diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index b8070d0be..9ce8ccfbe 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -44,21 +44,16 @@ async def close(self): # TODO: considering to create a BFTransportBuffer class to abstract the logic of binary buffers adapting to # current interfaces async def receive( - self, buffer: List[int], offset: int = None, count: int = None + self, buffer: List[int], offset: int = 0, count: int = None ) -> int: try: if self._socket: result = await self._socket.receive() - buffer_index = buffer.index(None) if None in buffer else 0 - result_index = 0 - while ( - buffer_index < len(buffer) - and result_index < len(result.data) - and result_index < count - ): + buffer_index = offset + result_length = count if count is not None else len(result.data) + for result_index in range(result_length): buffer[buffer_index] = result.data[result_index] buffer_index += 1 - result_index += 1 if result.message_type == WebSocketMessageType.CLOSE: await self._socket.close( WebSocketCloseStatus.NORMAL_CLOSURE, "Socket closed" @@ -68,7 +63,7 @@ async def receive( if self._socket.status == WebSocketState.CLOSED: self._socket.dispose() - return len(result.data) + return result_length except Exception as error: # Exceptions of the three types below will also have set the socket's state to closed, which fires an # event consumers of this class are subscribed to and have handling around. Any other exception needs to From a58323461e1403842cc75c3d5f2918e8b6b7a04c Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Fri, 23 Apr 2021 18:28:53 -0700 Subject: [PATCH 25/37] Fixes on the RecieveResponse path --- .../core/streaming/streaming_http_client.py | 4 +--- .../core/streaming/streaming_request_handler.py | 5 +---- .../streaming/payload_transport/payload_sender.py | 1 - .../botbuilder/streaming/receive_response.py | 1 + .../transport/web_socket/web_socket_transport.py | 11 ++++++----- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index 2277ae384..8606546a2 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -32,9 +32,7 @@ def body(self) -> bytes: """Return the whole body as bytes in memory. """ if not self._body: - raise ValueError( - "Body is not available. Call async method load_body, or do your call with stream=False." - ) + return bytes([]) return self._body async def load_body(self) -> None: diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index 4e10f1313..1bbbb9d48 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -199,10 +199,7 @@ async def send_streaming_request( "Error while attempting to send: Streaming transport is disconnected." ) - server_response = await self._server.send(request) - - if server_response.status_code == HTTPStatus.OK: - return server_response.read_body_as_json(ReceiveResponse) + return await self._server.send(request) except Exception: # TODO: remove printing and log it traceback.print_exc() diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py index e8838a1c3..ffb0351c1 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py @@ -138,7 +138,6 @@ async def _write_packet(self, packet: SendPacket): # TODO: this has to be improved in custom buffer class (validate buffer ended) for index in range(count): self._send_content_buffer[index] = packet.payload[index] - count = len(self._send_content_buffer) # Send: Packet content length = await self._sender.send( diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py index 23d7a25f4..1da08ee47 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py @@ -18,6 +18,7 @@ def read_body_as_json( ) -> Union[Model, Serializable]: try: body_str = self.read_body_as_str() + body = None if issubclass(cls, Serializable): body = cls().from_json(body_str) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py index 9ce8ccfbe..600e215a4 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py @@ -18,7 +18,6 @@ def __init__(self, web_socket: WebSocket): @property def is_connected(self): - print("Getting value") # TODO: mock logic return self._socket.status == WebSocketState.OPEN @@ -71,12 +70,14 @@ async def receive( raise error # TODO: might need to remove offset and count if no segmentation possible (or put them in BFTransportBuffer) - async def send( - self, buffer: List[int], offset: int = None, count: int = None - ) -> int: + async def send(self, buffer: List[int], offset: int = 0, count: int = None) -> int: try: if self._socket: - await self._socket.send(buffer, WebSocketMessageType.BINARY, True) + await self._socket.send( + buffer[offset:count] if count is not None else buffer, + WebSocketMessageType.BINARY, + True, + ) return count or len(buffer) except Exception as error: # Exceptions of the three types below will also have set the socket's state to closed, which fires an From f2991b6aa41158c4df8d2cafd1be75e4d42b5fc8 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Tue, 27 Apr 2021 18:10:28 -0700 Subject: [PATCH 26/37] PayloadStream length fix --- .../botbuilder-streaming/botbuilder/streaming/payload_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py index 9b1535d14..ce34e4eb8 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py @@ -20,7 +20,7 @@ def __init__(self, assembler: PayloadStreamAssembler): self._end = False def __len__(self): - return _producer_length + return self._producer_length def give_buffer(self, buffer: List[int]): self._buffer_queue.append(buffer) From 1e70c044875d3b2cdc5d6cc3d1bd367b0fd6c534 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 28 Apr 2021 13:07:01 -0700 Subject: [PATCH 27/37] Grouping related imports --- .../botbuilder/core/streaming/__init__.py | 3 - .../bot_framework_http_adapter_base.py | 51 +---------------- .../aiohttp/bot_framework_http_adapter.py | 55 ++++++++++++++++++- .../integration/aiohttp/streaming/__init__.py | 8 +++ .../aiohttp}/streaming/aiohttp_web_socket.py | 0 .../botbuilder-integration-aiohttp/setup.py | 1 + 6 files changed, 63 insertions(+), 55 deletions(-) create mode 100644 libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/__init__.py rename libraries/{botbuilder-core/botbuilder/core => botbuilder-integration-aiohttp/botbuilder/integration/aiohttp}/streaming/aiohttp_web_socket.py (100%) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py index bcb8890bb..b92ba04be 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/__init__.py @@ -1,15 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# TODO: this class is gonna be moved eventually to integration -from .aiohttp_web_socket import AiohttpWebSocket from .bot_framework_http_adapter_base import BotFrameworkHttpAdapterBase from .streaming_activity_processor import StreamingActivityProcessor from .streaming_request_handler import StreamingRequestHandler from .version_info import VersionInfo __all__ = [ - "AiohttpWebSocket", "BotFrameworkHttpAdapterBase", "StreamingActivityProcessor", "StreamingRequestHandler", diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py index 4e84c9632..86bd9246a 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/bot_framework_http_adapter_base.py @@ -4,7 +4,6 @@ from http import HTTPStatus from typing import Awaitable, Callable, List -from aiohttp import ClientSession from botbuilder.core import ( Bot, BotFrameworkAdapter, @@ -24,7 +23,6 @@ from .streaming_activity_processor import StreamingActivityProcessor from .streaming_request_handler import StreamingRequestHandler from .streaming_http_client import StreamingHttpDriver -from .aiohttp_web_socket import AiohttpWebSocket class BotFrameworkHttpAdapterBase(BotFrameworkAdapter, StreamingActivityProcessor): @@ -80,54 +78,7 @@ async def process_streaming_activity( return None async def send_streaming_activity(self, activity: Activity) -> ResourceResponse: - # Check to see if any of this adapter's StreamingRequestHandlers is associated with this conversation. - possible_handlers = [ - handler - for handler in self.request_handlers - if handler.service_url == activity.service_url - and handler.has_conversation(activity.conversation.id) - ] - - if possible_handlers: - if len(possible_handlers) > 1: - # The conversation has moved to a new connection and the former - # StreamingRequestHandler needs to be told to forget about it. - possible_handlers.sort( - key=lambda handler: handler.conversation_added_time( - activity.conversation.id - ) - ) - correct_handler = possible_handlers[-1] - for handler in possible_handlers: - if handler is not correct_handler: - handler.forget_conversation(activity.conversation.id) - - return await correct_handler.send_activity(activity) - - return await possible_handlers[0].send_activity(activity) - - if self.connected_bot: - # This is a proactive message that will need a new streaming connection opened. - # The ServiceUrl of a streaming connection follows the pattern "url:[ChannelName]:[Protocol]:[Host]". - - uri = activity.service_url.split(":") - protocol = uri[len(uri) - 2] - host = uri[len(uri) - 1] - # TODO: discuss if should abstract this from current package - # TODO: manage life cycle of sessions (when should we close them) - session = ClientSession() - aiohttp_ws = await session.ws_connect(protocol + host + "/api/messages") - web_socket = AiohttpWebSocket(aiohttp_ws, session) - handler = StreamingRequestHandler(self.connected_bot, self, web_socket) - - if self.request_handlers is None: - self.request_handlers = [] - - self.request_handlers.append(handler) - - return await handler.send_activity(activity) - - return None + raise NotImplementedError() def can_process_outgoing_activity(self, activity: Activity) -> bool: if not activity: diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py index 09e1507e1..da1b7c3c3 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/bot_framework_http_adapter.py @@ -3,6 +3,7 @@ from typing import Optional +from aiohttp import ClientSession from aiohttp.web import ( Request, Response, @@ -14,11 +15,11 @@ ) from botbuilder.core import Bot, BotFrameworkAdapterSettings from botbuilder.core.streaming import ( - AiohttpWebSocket, BotFrameworkHttpAdapterBase, StreamingRequestHandler, ) -from botbuilder.schema import Activity +from botbuilder.schema import Activity, ResourceResponse +from botbuilder.integration.aiohttp.streaming import AiohttpWebSocket from botframework.connector.auth import AuthenticationConstants, JwtTokenValidation @@ -67,6 +68,56 @@ async def process( ) return Response(status=201) + async def send_streaming_activity(self, activity: Activity) -> ResourceResponse: + # Check to see if any of this adapter's StreamingRequestHandlers is associated with this conversation. + possible_handlers = [ + handler + for handler in self.request_handlers + if handler.service_url == activity.service_url + and handler.has_conversation(activity.conversation.id) + ] + + if possible_handlers: + if len(possible_handlers) > 1: + # The conversation has moved to a new connection and the former + # StreamingRequestHandler needs to be told to forget about it. + possible_handlers.sort( + key=lambda handler: handler.conversation_added_time( + activity.conversation.id + ) + ) + correct_handler = possible_handlers[-1] + for handler in possible_handlers: + if handler is not correct_handler: + handler.forget_conversation(activity.conversation.id) + + return await correct_handler.send_activity(activity) + + return await possible_handlers[0].send_activity(activity) + + if self.connected_bot: + # This is a proactive message that will need a new streaming connection opened. + # The ServiceUrl of a streaming connection follows the pattern "url:[ChannelName]:[Protocol]:[Host]". + + uri = activity.service_url.split(":") + protocol = uri[len(uri) - 2] + host = uri[len(uri) - 1] + # TODO: discuss if should abstract this from current package + # TODO: manage life cycle of sessions (when should we close them) + session = ClientSession() + aiohttp_ws = await session.ws_connect(protocol + host + "/api/messages") + web_socket = AiohttpWebSocket(aiohttp_ws, session) + handler = StreamingRequestHandler(self.connected_bot, self, web_socket) + + if self.request_handlers is None: + self.request_handlers = [] + + self.request_handlers.append(handler) + + return await handler.send_activity(activity) + + return None + async def _connect_web_socket( self, bot: Bot, request: Request, ws_response: WebSocketResponse ): diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/__init__.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/__init__.py new file mode 100644 index 000000000..4d380bf47 --- /dev/null +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .aiohttp_web_socket import AiohttpWebSocket + +__all__ = [ + "AiohttpWebSocket", +] diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py similarity index 100% rename from libraries/botbuilder-core/botbuilder/core/streaming/aiohttp_web_socket.py rename to libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py diff --git a/libraries/botbuilder-integration-aiohttp/setup.py b/libraries/botbuilder-integration-aiohttp/setup.py index 69605aadd..feb556f4a 100644 --- a/libraries/botbuilder-integration-aiohttp/setup.py +++ b/libraries/botbuilder-integration-aiohttp/setup.py @@ -42,6 +42,7 @@ packages=[ "botbuilder.integration.aiohttp", "botbuilder.integration.aiohttp.skills", + "botbuilder.integration.aiohttp.streaming", ], install_requires=REQUIRES, classifiers=[ From c7b8fafc17653c764b0e362031ee266e0229bd0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Axel=20Su=C3=A1rez?= Date: Fri, 30 Apr 2021 16:57:12 -0700 Subject: [PATCH 28/37] payload receiver unit test (#1664) --- .../payload_transport/payload_receiver.py | 1 + .../streaming/payload_transport/send_queue.py | 2 +- .../tests/test_payload_receiver.py | 70 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 libraries/botbuilder-streaming/tests/test_payload_receiver.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py index df054ced8..092c0f1cb 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py @@ -88,6 +88,7 @@ async def disconnect(self, event_args: DisconnectedEventArgs = None): async def _receive_packets(self): is_closed = False + disconnect_args = None while self._receiver and self._receiver.is_connected and not is_closed: # receive a single packet diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py index 07d197496..4163e7558 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py +++ b/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py @@ -27,7 +27,7 @@ async def _process(self): while True: try: while True: - await sleep(1) + await sleep(0.2) item = await self._queue.get() try: await self._action(item) diff --git a/libraries/botbuilder-streaming/tests/test_payload_receiver.py b/libraries/botbuilder-streaming/tests/test_payload_receiver.py new file mode 100644 index 000000000..315bd23a7 --- /dev/null +++ b/libraries/botbuilder-streaming/tests/test_payload_receiver.py @@ -0,0 +1,70 @@ +from typing import List + +import aiounittest + +from botbuilder.streaming import PayloadStream +from botbuilder.streaming.payload_transport import PayloadReceiver +from botbuilder.streaming.transport import TransportReceiverBase + + +class MockTransportReceiver(TransportReceiverBase): + def __init__(self, mock_header: bytes, mock_payload: bytes): + self._is_connected = True + self._mock_gen = self._mock_receive(mock_header, mock_payload) + + def _mock_receive(self, mock_header: bytes, mock_payload: bytes): + yield mock_header + yield mock_payload + + @property + def is_connected(self): + if self._is_connected: + self._is_connected = False + return True + return False + + async def close(self): + return + + async def receive(self, buffer: object, offset: int, count: int) -> int: + resp_buffer = list(next(self._mock_gen)) + for index, val in enumerate(resp_buffer): + buffer[index] = val + return len(resp_buffer) + + +class MockStream(PayloadStream): + # pylint: disable=super-init-not-called + def __init__(self): + self.buffer = None + self._producer_length = 0 # total length + + def give_buffer(self, buffer: List[int]): + self.buffer = buffer + + +class TestBotFrameworkHttpClient(aiounittest.AsyncTestCase): + async def test_connect(self): + mock_header = b"S.000004.e35ed534-0808-4acf-af1e-24aa81d2b31d.1\n" + mock_payload = b"test" + + mock_receiver = MockTransportReceiver(mock_header, mock_payload) + mock_stream = MockStream() + + receive_action_called = False + + def mock_get_stream(header): # pylint: disable=unused-argument + return mock_stream + + def mock_receive_action(header, stream, offset): + nonlocal receive_action_called + assert header.type == "S" + assert len(stream.buffer) == offset + receive_action_called = True + + sut = PayloadReceiver() + sut.subscribe(mock_get_stream, mock_receive_action) + await sut.connect(mock_receiver) + + assert bytes(mock_stream.buffer) == mock_payload + assert receive_action_called From f0de9a0925d3c1726019c6ccfc202eb792410082 Mon Sep 17 00:00:00 2001 From: Muthuveer Somanathan <41929942+msomanathan@users.noreply.github.com> Date: Mon, 3 May 2021 11:41:10 -0700 Subject: [PATCH 29/37] Payload sender unit test (#1666) * payload receiver unit test * senderTest * disconnect * fixWarning Co-authored-by: Axel Suarez --- .../tests/test_payload_sender.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 libraries/botbuilder-streaming/tests/test_payload_sender.py diff --git a/libraries/botbuilder-streaming/tests/test_payload_sender.py b/libraries/botbuilder-streaming/tests/test_payload_sender.py new file mode 100644 index 000000000..75528b8ba --- /dev/null +++ b/libraries/botbuilder-streaming/tests/test_payload_sender.py @@ -0,0 +1,56 @@ +from asyncio import Semaphore +from typing import List +from uuid import UUID, uuid4 + +import aiounittest +from botbuilder.streaming.payload_transport import PayloadSender +from botbuilder.streaming.payloads import HeaderSerializer +from botbuilder.streaming.payloads.models import Header +from botbuilder.streaming.transport import TransportSenderBase + + +class MockTransportSender(TransportSenderBase): + def __init__(self): + super().__init__() + self.send_called = Semaphore(0) + + async def send(self, buffer: List[int], offset: int, count: int) -> int: + # Assert + if count == 48: # Header + print("Validating Header...") + header = HeaderSerializer.deserialize(buffer, offset, count) + assert header.type == "A" + assert header.payload_length == 3 + assert header.end + else: # Payload + print("Validating Payload...") + assert count == 3 + self.send_called.release() + + return count + + def close(self): + pass + + +class TestPayloadSender(aiounittest.AsyncTestCase): + async def test_send(self): + # Arrange + sut = PayloadSender() + sender = MockTransportSender() + sut.connect(sender) + + headerId: UUID = uuid4() + header = Header(type="A", id=headerId, end=True) + header.payload_length = 3 + payload = [1, 2, 3] + + async def mock_sent_callback(h: Header): + print(f"{h.type}.{h.payload_length}.{h.id}.{h.end}") + + # Act + sut.send_payload(header, payload, is_length_known=True, sent_callback=mock_sent_callback) + + # Assert + await sender.send_called.acquire() + await sut.disconnect() From 7e68cb1f66ee34c0a551088686e127274575d318 Mon Sep 17 00:00:00 2001 From: msomanathan Date: Mon, 3 May 2021 11:56:56 -0700 Subject: [PATCH 30/37] blackcheck --- libraries/botbuilder-streaming/tests/test_payload_sender.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/botbuilder-streaming/tests/test_payload_sender.py b/libraries/botbuilder-streaming/tests/test_payload_sender.py index 75528b8ba..cf2edd2c8 100644 --- a/libraries/botbuilder-streaming/tests/test_payload_sender.py +++ b/libraries/botbuilder-streaming/tests/test_payload_sender.py @@ -49,7 +49,9 @@ async def mock_sent_callback(h: Header): print(f"{h.type}.{h.payload_length}.{h.id}.{h.end}") # Act - sut.send_payload(header, payload, is_length_known=True, sent_callback=mock_sent_callback) + sut.send_payload( + header, payload, is_length_known=True, sent_callback=mock_sent_callback + ) # Assert await sender.send_called.acquire() From ad70fa5832596117c68c2e09eab3321207968dee Mon Sep 17 00:00:00 2001 From: msomanathan Date: Mon, 3 May 2021 12:07:29 -0700 Subject: [PATCH 31/37] pylintfix --- .../botbuilder-streaming/tests/test_payload_sender.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/botbuilder-streaming/tests/test_payload_sender.py b/libraries/botbuilder-streaming/tests/test_payload_sender.py index cf2edd2c8..5aaeb880d 100644 --- a/libraries/botbuilder-streaming/tests/test_payload_sender.py +++ b/libraries/botbuilder-streaming/tests/test_payload_sender.py @@ -40,13 +40,15 @@ async def test_send(self): sender = MockTransportSender() sut.connect(sender) - headerId: UUID = uuid4() - header = Header(type="A", id=headerId, end=True) + header_id: UUID = uuid4() + header = Header(type="A", id=header_id, end=True) header.payload_length = 3 payload = [1, 2, 3] - async def mock_sent_callback(h: Header): - print(f"{h.type}.{h.payload_length}.{h.id}.{h.end}") + async def mock_sent_callback(callback_header: Header): + print( + f"{callback_header.type}.{callback_header.payload_length}.{callback_header.id}.{callback_header.end}" + ) # Act sut.send_payload( From 9601b2e562dddec5a907964423c0499ea410ec87 Mon Sep 17 00:00:00 2001 From: Muthuveer Somanathan <41929942+msomanathan@users.noreply.github.com> Date: Wed, 5 May 2021 15:31:12 -0700 Subject: [PATCH 32/37] test_req_processor (#1668) --- .../tests/test_payload_processor.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 libraries/botbuilder-streaming/tests/test_payload_processor.py diff --git a/libraries/botbuilder-streaming/tests/test_payload_processor.py b/libraries/botbuilder-streaming/tests/test_payload_processor.py new file mode 100644 index 000000000..0f43bcbd6 --- /dev/null +++ b/libraries/botbuilder-streaming/tests/test_payload_processor.py @@ -0,0 +1,47 @@ +from typing import List +from uuid import UUID, uuid4 + +import aiounittest +from botbuilder.streaming import ReceiveRequest +from botbuilder.streaming.payloads import StreamManager +from botbuilder.streaming.payloads.assemblers import ReceiveRequestAssembler, PayloadStreamAssembler +from botbuilder.streaming.payloads.models import Header, RequestPayload, StreamDescription + + +class MockStreamManager(StreamManager): + def __init__(self): + super().__init__() + + def get_payload_assembler(self, identifier: UUID) -> PayloadStreamAssembler: + return PayloadStreamAssembler(self, identifier) + + +class TestPayloadProcessor(aiounittest.AsyncTestCase): + async def test_process_request(self): + # Arrange + header_id: UUID = uuid4() + header = Header(type="A", id=header_id, end=True) + header.payload_length = 3 + stream_manager = MockStreamManager() + + on_completed_called = False + + async def mock_on_completed(identifier: UUID, request: ReceiveRequest): + nonlocal on_completed_called + assert identifier == header_id + assert request.verb == "POST" + assert request.path == "/api/messages" + assert len(request.streams) == 1 + on_completed_called = True + + sut = ReceiveRequestAssembler(header, stream_manager, on_completed=mock_on_completed) + + # Act + stream_id: UUID = uuid4() + streams: List[StreamDescription] = [StreamDescription(id=str(stream_id), content_type="json", length=100)] + payload = RequestPayload(verb="POST", path="/api/messages", streams=streams).to_json() + payload_stream: List[int] = list(bytes(payload, "utf-8")) + await sut.process_request(payload_stream) + + # Assert + assert on_completed_called From 004dfff6cfe4349ecadfa4f67a6f0b6026bd29f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Axel=20Su=C3=A1rez?= Date: Wed, 5 May 2021 15:31:30 -0700 Subject: [PATCH 33/37] Axsuarez/streaming receive loop unittest (#1667) * payload receiver unit test * StreamingRequestHandler test listen --- .../test_streaming_request_handler.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py diff --git a/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py b/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py new file mode 100644 index 000000000..4c0cfe995 --- /dev/null +++ b/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from unittest.mock import Mock +from typing import Any + +import aiounittest + +from botbuilder.core.streaming import StreamingRequestHandler +from botbuilder.streaming.transport.web_socket import ( + WebSocket, + WebSocketState, + WebSocketCloseStatus, + WebSocketMessage, + WebSocketMessageType, +) + + +class MockWebSocket(WebSocket): + def __init__(self): + super(MockWebSocket, self).__init__() + + self.receive_called = False + + def dispose(self): + return + + async def close(self, close_status: WebSocketCloseStatus, status_description: str): + return + + async def receive(self) -> WebSocketMessage: + self.receive_called = True + + async def send( + self, buffer: Any, message_type: WebSocketMessageType, end_of_message: bool + ): + raise Exception + + @property + def status(self) -> WebSocketState: + return WebSocketState.OPEN + + +class TestStramingRequestHandler(aiounittest.AsyncTestCase): + async def test_listen(self): + mock_bot = Mock() + mock_activity_processor = Mock() + mock_web_socket = MockWebSocket() + + sut = StreamingRequestHandler( + mock_bot, mock_activity_processor, mock_web_socket + ) + await sut.listen() + + assert mock_web_socket.receive_called From 6d23924f40fc11a74695fee8a5e7d911f2adceb3 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Wed, 5 May 2021 19:26:55 -0700 Subject: [PATCH 34/37] renaming straming to botframework scope --- .../core/streaming/streaming_http_client.py | 2 +- .../streaming/streaming_request_handler.py | 6 ++-- .../botbuilder/core/streaming/version_info.py | 2 +- .../test_streaming_request_handler.py | 2 +- .../aiohttp/streaming/aiohttp_web_socket.py | 2 +- .../README.rst | 16 +++++------ .../botframework}/streaming/__init__.py | 0 .../botframework}/streaming/about.py | 28 +++++++++---------- .../botframework}/streaming/payload_stream.py | 2 +- .../streaming/payload_transport/__init__.py | 0 .../payload_transport/payload_receiver.py | 8 +++--- .../payload_transport/payload_sender.py | 6 ++-- .../payload_transport/send_packet.py | 2 +- .../streaming/payload_transport/send_queue.py | 0 .../streaming/payloads/__init__.py | 0 .../streaming/payloads/assemblers/__init__.py | 0 .../payloads/assemblers/assembler.py | 2 +- .../assemblers/payload_stream_assembler.py | 6 ++-- .../assemblers/receive_request_assembler.py | 6 ++-- .../assemblers/receive_response_assembler.py | 6 ++-- .../streaming/payloads/content_stream.py | 2 +- .../payloads/disassemblers/__init__.py | 0 .../disassemblers/cancel_disassembler.py | 4 +-- .../disassemblers/payload_disassembler.py | 12 +++++--- .../disassemblers/request_disassembler.py | 4 +-- .../disassemblers/response_disassembler.py | 4 +-- .../response_message_stream_disassembler.py | 6 ++-- .../streaming/payloads/header_serializer.py | 2 +- .../streaming/payloads/models/__init__.py | 0 .../streaming/payloads/models/header.py | 2 +- .../payloads/models/payload_types.py | 0 .../payloads/models/request_payload.py | 0 .../payloads/models/response_payload.py | 0 .../streaming/payloads/models/serializable.py | 0 .../payloads/models/stream_description.py | 0 .../payloads/payload_assembler_manager.py | 4 +-- .../streaming/payloads/request_manager.py | 2 +- .../payloads/response_message_stream.py | 0 .../streaming/payloads/send_operations.py | 6 ++-- .../streaming/payloads/stream_manager.py | 4 +-- .../streaming/protocol_adapter.py | 6 ++-- .../streaming/receive_request.py | 2 +- .../streaming/receive_response.py | 4 +-- .../streaming/request_handler.py | 0 .../streaming/streaming_request.py | 4 +-- .../streaming/streaming_response.py | 4 +-- .../streaming/transport/__init__.py | 0 .../transport/disconnected_event_args.py | 0 .../transport/streaming_transport_service.py | 0 .../streaming/transport/transport_base.py | 0 .../transport/transport_constants.py | 0 .../transport/transport_receiver_base.py | 0 .../transport/transport_sender_base.py | 0 .../transport/web_socket/__init__.py | 0 .../transport/web_socket/web_socket.py | 0 .../web_socket/web_socket_close_status.py | 0 .../web_socket/web_socket_message_type.py | 0 .../transport/web_socket/web_socket_server.py | 8 +++--- .../transport/web_socket/web_socket_state.py | 0 .../web_socket/web_socket_transport.py | 2 +- .../requirements.txt | 0 .../setup.py | 16 +++++------ .../tests/test_payload_processor.py | 27 +++++++++++++----- .../tests/test_payload_receiver.py | 6 ++-- .../tests/test_payload_sender.py | 8 +++--- 65 files changed, 126 insertions(+), 109 deletions(-) rename libraries/{botbuilder-streaming => botframework-streaming}/README.rst (85%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/about.py (86%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_stream.py (96%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_transport/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_transport/payload_receiver.py (96%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_transport/payload_sender.py (97%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_transport/send_packet.py (89%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payload_transport/send_queue.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/assemblers/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/assemblers/assembler.py (92%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/assemblers/payload_stream_assembler.py (89%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/assemblers/receive_request_assembler.py (94%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/assemblers/receive_response_assembler.py (94%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/content_stream.py (89%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/cancel_disassembler.py (81%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/payload_disassembler.py (91%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/request_disassembler.py (88%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/response_disassembler.py (88%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/disassemblers/response_message_stream_disassembler.py (79%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/header_serializer.py (98%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/header.py (93%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/payload_types.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/request_payload.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/response_payload.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/serializable.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/models/stream_description.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/payload_assembler_manager.py (95%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/request_manager.py (96%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/response_message_stream.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/send_operations.py (91%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/payloads/stream_manager.py (92%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/protocol_adapter.py (93%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/receive_request.py (93%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/receive_response.py (92%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/request_handler.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/streaming_request.py (94%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/streaming_response.py (91%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/disconnected_event_args.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/streaming_transport_service.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/transport_base.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/transport_constants.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/transport_receiver_base.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/transport_sender_base.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/__init__.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket_close_status.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket_message_type.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket_server.py (92%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket_state.py (100%) rename libraries/{botbuilder-streaming/botbuilder => botframework-streaming/botframework}/streaming/transport/web_socket/web_socket_transport.py (97%) rename libraries/{botbuilder-streaming => botframework-streaming}/requirements.txt (100%) rename libraries/{botbuilder-streaming => botframework-streaming}/setup.py (74%) rename libraries/{botbuilder-streaming => botframework-streaming}/tests/test_payload_processor.py (62%) rename libraries/{botbuilder-streaming => botframework-streaming}/tests/test_payload_receiver.py (91%) rename libraries/{botbuilder-streaming => botframework-streaming}/tests/test_payload_sender.py (86%) diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py index 8606546a2..9d80e63e6 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_http_client.py @@ -10,7 +10,7 @@ from msrest.universal_http.async_requests import ( AsyncRequestsHTTPSender as AsyncRequestsHTTPDriver, ) -from botbuilder.streaming import StreamingRequest, ReceiveResponse +from botframework.streaming import StreamingRequest, ReceiveResponse from .streaming_request_handler import StreamingRequestHandler diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py index 1bbbb9d48..9b9ac4e8d 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/streaming_request_handler.py @@ -11,7 +11,7 @@ from botbuilder.core import Bot from botbuilder.schema import Activity, Attachment, ResourceResponse -from botbuilder.streaming import ( +from botframework.streaming import ( RequestHandler, ReceiveRequest, ReceiveResponse, @@ -20,8 +20,8 @@ __title__, __version__, ) -from botbuilder.streaming.transport import DisconnectedEventArgs -from botbuilder.streaming.transport.web_socket import WebSocket, WebSocketServer +from botframework.streaming.transport import DisconnectedEventArgs +from botframework.streaming.transport.web_socket import WebSocket, WebSocketServer from .streaming_activity_processor import StreamingActivityProcessor from .version_info import VersionInfo diff --git a/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py b/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py index f866b8b7d..b11250375 100644 --- a/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py +++ b/libraries/botbuilder-core/botbuilder/core/streaming/version_info.py @@ -3,7 +3,7 @@ import json -from botbuilder.streaming.payloads.models import Serializable +from botframework.streaming.payloads.models import Serializable class VersionInfo(Serializable): diff --git a/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py b/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py index 4c0cfe995..39e6802a6 100644 --- a/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py +++ b/libraries/botbuilder-core/tests/streaming/test_streaming_request_handler.py @@ -7,7 +7,7 @@ import aiounittest from botbuilder.core.streaming import StreamingRequestHandler -from botbuilder.streaming.transport.web_socket import ( +from botframework.streaming.transport.web_socket import ( WebSocket, WebSocketState, WebSocketCloseStatus, diff --git a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py index 5e351eb70..334c637fb 100644 --- a/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py +++ b/libraries/botbuilder-integration-aiohttp/botbuilder/integration/aiohttp/streaming/aiohttp_web_socket.py @@ -9,7 +9,7 @@ from aiohttp import ClientWebSocketResponse, WSMsgType, ClientSession from aiohttp.web import WebSocketResponse -from botbuilder.streaming.transport.web_socket import ( +from botframework.streaming.transport.web_socket import ( WebSocket, WebSocketMessage, WebSocketCloseStatus, diff --git a/libraries/botbuilder-streaming/README.rst b/libraries/botframework-streaming/README.rst similarity index 85% rename from libraries/botbuilder-streaming/README.rst rename to libraries/botframework-streaming/README.rst index a2bfe44c7..49595961f 100644 --- a/libraries/botbuilder-streaming/README.rst +++ b/libraries/botframework-streaming/README.rst @@ -1,30 +1,30 @@ =============================== -BotBuilder-Streaming for Python +BotFramework-Streaming for Python =============================== .. image:: https://fuselabs.visualstudio.com/SDK_v4/_apis/build/status/Python/SDK_v4-Python-CI?branchName=master :target: https://fuselabs.visualstudio.com/SDK_v4/_apis/build/status/Python/SDK_v4-Python-CI :align: right :alt: Azure DevOps status for master branch -.. image:: https://badge.fury.io/py/botbuilder-core.svg - :target: https://badge.fury.io/py/botbuilder-core +.. image:: https://badge.fury.io/py/botframework-streaming.svg + :target: https://badge.fury.io/py/botframework-streaming :alt: Latest PyPI package version -Streaming Extensions libraries for BotBuilder. +Streaming Extensions libraries for BotFramework. How to Install ============== .. code-block:: python - pip install botbuilder-streaming + pip install botframework-streaming Documentation/Wiki ================== -You can find more information on the botbuilder-python project by visiting our `Wiki`_. +You can find more information on the botframework-python project by visiting our `Wiki`_. Requirements ============ @@ -35,7 +35,7 @@ Requirements Source Code =========== The latest developer version is available in a github repository: -https://github.com/Microsoft/botbuilder-python/ +https://github.com/Microsoft/botframework-python/ Contributing @@ -70,7 +70,7 @@ Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT_ License. -.. _Wiki: https://github.com/Microsoft/botbuilder-python/wiki +.. _Wiki: https://github.com/Microsoft/botframework-python/wiki .. _Python >= 3.7.0: https://www.python.org/downloads/ .. _MIT: https://github.com/Microsoft/vscode/blob/master/LICENSE.txt .. _Microsoft Open Source Code of Conduct: https://opensource.microsoft.com/codeofconduct/ diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/__init__.py b/libraries/botframework-streaming/botframework/streaming/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/__init__.py rename to libraries/botframework-streaming/botframework/streaming/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/about.py b/libraries/botframework-streaming/botframework/streaming/about.py similarity index 86% rename from libraries/botbuilder-streaming/botbuilder/streaming/about.py rename to libraries/botframework-streaming/botframework/streaming/about.py index 3cd35b078..81e170270 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/about.py +++ b/libraries/botframework-streaming/botframework/streaming/about.py @@ -1,14 +1,14 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import os - -__title__ = "botbuilder-streaming" -__version__ = ( - os.environ["packageVersion"] if "packageVersion" in os.environ else "4.7.1" -) -__uri__ = "https://www.github.com/Microsoft/botbuilder-python" -__author__ = "Microsoft" -__description__ = "Microsoft Bot Framework Bot Builder" -__summary__ = "Microsoft Bot Framework Bot Builder SDK for Python." -__license__ = "MIT" +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os + +__title__ = "botframework-streaming" +__version__ = ( + os.environ["packageVersion"] if "packageVersion" in os.environ else "4.13.0" +) +__uri__ = "https://www.github.com/Microsoft/botbuilder-python" +__author__ = "Microsoft" +__description__ = "Microsoft Bot Framework Bot Builder" +__summary__ = "Microsoft Bot Framework Bot Builder SDK for Python." +__license__ = "MIT" diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py b/libraries/botframework-streaming/botframework/streaming/payload_stream.py similarity index 96% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py rename to libraries/botframework-streaming/botframework/streaming/payload_stream.py index ce34e4eb8..4a9ec1463 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_stream.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_stream.py @@ -4,7 +4,7 @@ from asyncio import Lock, Semaphore from typing import List -from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler +from botframework.streaming.payloads.assemblers import PayloadStreamAssembler class PayloadStream: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/__init__.py rename to libraries/botframework-streaming/botframework/streaming/payload_transport/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/payload_receiver.py similarity index 96% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py rename to libraries/botframework-streaming/botframework/streaming/payload_transport/payload_receiver.py index 092c0f1cb..b20df2050 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_receiver.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_transport/payload_receiver.py @@ -6,10 +6,10 @@ from asyncio import iscoroutinefunction, isfuture from typing import Callable, List -import botbuilder.streaming as streaming -from botbuilder.streaming.payloads import HeaderSerializer -from botbuilder.streaming.payloads.models import Header, PayloadTypes -from botbuilder.streaming.transport import ( +import botframework.streaming as streaming +from botframework.streaming.payloads import HeaderSerializer +from botframework.streaming.payloads.models import Header, PayloadTypes +from botframework.streaming.transport import ( DisconnectedEventArgs, TransportConstants, TransportReceiverBase, diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/payload_sender.py similarity index 97% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py rename to libraries/botframework-streaming/botframework/streaming/payload_transport/payload_sender.py index ffb0351c1..817181846 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/payload_sender.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_transport/payload_sender.py @@ -4,13 +4,13 @@ from asyncio import Event, ensure_future, iscoroutinefunction, isfuture from typing import Awaitable, Callable, List -from botbuilder.streaming.transport import ( +from botframework.streaming.transport import ( DisconnectedEventArgs, TransportSenderBase, TransportConstants, ) -from botbuilder.streaming.payloads import HeaderSerializer -from botbuilder.streaming.payloads.models import Header +from botframework.streaming.payloads import HeaderSerializer +from botframework.streaming.payloads.models import Header from .send_queue import SendQueue from .send_packet import SendPacket diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_packet.py similarity index 89% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py rename to libraries/botframework-streaming/botframework/streaming/payload_transport/send_packet.py index 08e4a770e..bf7164708 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_packet.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_packet.py @@ -3,7 +3,7 @@ from typing import Awaitable, Callable -from botbuilder.streaming.payloads.models import Header +from botframework.streaming.payloads.models import Header class SendPacket: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payload_transport/send_queue.py rename to libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py b/libraries/botframework-streaming/botframework/streaming/payloads/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/__init__.py rename to libraries/botframework-streaming/botframework/streaming/payloads/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/__init__.py rename to libraries/botframework-streaming/botframework/streaming/payloads/assemblers/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/assembler.py similarity index 92% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/assemblers/assembler.py index 3a0c048d2..5fdcfe49d 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/assembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/assembler.py @@ -7,7 +7,7 @@ from typing import List -from botbuilder.streaming.payloads.models import Header +from botframework.streaming.payloads.models import Header class Assembler(ABC): diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/payload_stream_assembler.py similarity index 89% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/assemblers/payload_stream_assembler.py index 6f19799cd..c7aba13b6 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/payload_stream_assembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/payload_stream_assembler.py @@ -4,9 +4,9 @@ from uuid import UUID from typing import List -import botbuilder.streaming as streaming -import botbuilder.streaming.payloads as payloads -from botbuilder.streaming.payloads.models import Header +import botframework.streaming as streaming +import botframework.streaming.payloads as payloads +from botframework.streaming.payloads.models import Header from .assembler import Assembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_request_assembler.py similarity index 94% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_request_assembler.py index 0a78acd71..f26f67c6a 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_request_assembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_request_assembler.py @@ -5,9 +5,9 @@ from uuid import UUID from typing import Awaitable, Callable, List -import botbuilder.streaming as streaming -import botbuilder.streaming.payloads as payloads -from botbuilder.streaming.payloads.models import Header, RequestPayload +import botframework.streaming as streaming +import botframework.streaming.payloads as payloads +from botframework.streaming.payloads.models import Header, RequestPayload from .assembler import Assembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_response_assembler.py similarity index 94% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_response_assembler.py index f5602e508..9b6003021 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/assemblers/receive_response_assembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/assemblers/receive_response_assembler.py @@ -5,9 +5,9 @@ from uuid import UUID from typing import Awaitable, Callable, List -import botbuilder.streaming as streaming -import botbuilder.streaming.payloads as payloads -from botbuilder.streaming.payloads.models import Header, ResponsePayload +import botframework.streaming as streaming +import botframework.streaming.payloads as payloads +from botframework.streaming.payloads.models import Header, ResponsePayload from .assembler import Assembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py b/libraries/botframework-streaming/botframework/streaming/payloads/content_stream.py similarity index 89% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py rename to libraries/botframework-streaming/botframework/streaming/payloads/content_stream.py index f61b44764..c0c1ef67c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/content_stream.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/content_stream.py @@ -3,7 +3,7 @@ from uuid import UUID -from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler +from botframework.streaming.payloads.assemblers import PayloadStreamAssembler class ContentStream: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/__init__.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/cancel_disassembler.py similarity index 81% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/cancel_disassembler.py index f1c2f49eb..c531cfe5d 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/cancel_disassembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/cancel_disassembler.py @@ -3,8 +3,8 @@ from uuid import UUID -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads.models import Header +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads.models import Header class CancelDisassembler: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/payload_disassembler.py similarity index 91% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/payload_disassembler.py index 67f440d84..d60955d1f 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/payload_disassembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/payload_disassembler.py @@ -7,10 +7,14 @@ from uuid import UUID from typing import List -from botbuilder.streaming.transport import TransportConstants -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads import ResponseMessageStream -from botbuilder.streaming.payloads.models import Header, Serializable, StreamDescription +from botframework.streaming.transport import TransportConstants +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads import ResponseMessageStream +from botframework.streaming.payloads.models import ( + Header, + Serializable, + StreamDescription, +) class PayloadDisassembler(ABC): diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/request_disassembler.py similarity index 88% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/request_disassembler.py index 2cbccc211..281dec376 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/request_disassembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/request_disassembler.py @@ -4,8 +4,8 @@ from uuid import UUID from typing import List -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads.models import PayloadTypes, RequestPayload +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads.models import PayloadTypes, RequestPayload from .payload_disassembler import PayloadDisassembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_disassembler.py similarity index 88% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_disassembler.py index 038b4c0b8..7e480cac4 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_disassembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_disassembler.py @@ -4,8 +4,8 @@ from uuid import UUID from typing import List -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads.models import PayloadTypes, ResponsePayload +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads.models import PayloadTypes, ResponsePayload from .payload_disassembler import PayloadDisassembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_message_stream_disassembler.py similarity index 79% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py rename to libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_message_stream_disassembler.py index b25356df0..3f0f5d71c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/disassemblers/response_message_stream_disassembler.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/disassemblers/response_message_stream_disassembler.py @@ -3,9 +3,9 @@ from typing import List -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads import ResponseMessageStream -from botbuilder.streaming.payloads.models import PayloadTypes +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads import ResponseMessageStream +from botframework.streaming.payloads.models import PayloadTypes from .payload_disassembler import PayloadDisassembler diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py b/libraries/botframework-streaming/botframework/streaming/payloads/header_serializer.py similarity index 98% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py rename to libraries/botframework-streaming/botframework/streaming/payloads/header_serializer.py index fc2348c98..b0b507ab2 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/header_serializer.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/header_serializer.py @@ -4,7 +4,7 @@ from uuid import UUID from typing import List -from botbuilder.streaming.transport import TransportConstants +from botframework.streaming.transport import TransportConstants from .models import Header diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/__init__.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/header.py similarity index 93% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/header.py index 135749309..5eab7564e 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/header.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/models/header.py @@ -3,7 +3,7 @@ from uuid import UUID -from botbuilder.streaming.transport import TransportConstants +from botframework.streaming.transport import TransportConstants class Header: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/payload_types.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/payload_types.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/payload_types.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/request_payload.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/request_payload.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/request_payload.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/response_payload.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/response_payload.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/response_payload.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/serializable.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/serializable.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/serializable.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py b/libraries/botframework-streaming/botframework/streaming/payloads/models/stream_description.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/models/stream_description.py rename to libraries/botframework-streaming/botframework/streaming/payloads/models/stream_description.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py b/libraries/botframework-streaming/botframework/streaming/payloads/payload_assembler_manager.py similarity index 95% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py rename to libraries/botframework-streaming/botframework/streaming/payloads/payload_assembler_manager.py index 823efa536..276654b0b 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/payload_assembler_manager.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/payload_assembler_manager.py @@ -4,12 +4,12 @@ from uuid import UUID from typing import Awaitable, Callable, Dict, List, Union -from botbuilder.streaming.payloads.assemblers import ( +from botframework.streaming.payloads.assemblers import ( Assembler, ReceiveRequestAssembler, ReceiveResponseAssembler, ) -from botbuilder.streaming.payloads.models import Header, PayloadTypes +from botframework.streaming.payloads.models import Header, PayloadTypes from .stream_manager import StreamManager diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py b/libraries/botframework-streaming/botframework/streaming/payloads/request_manager.py similarity index 96% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py rename to libraries/botframework-streaming/botframework/streaming/payloads/request_manager.py index ccec9a176..0ffdbeaad 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/request_manager.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/request_manager.py @@ -5,7 +5,7 @@ from uuid import UUID from typing import Dict -import botbuilder.streaming as streaming +import botframework.streaming as streaming class RequestManager: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py b/libraries/botframework-streaming/botframework/streaming/payloads/response_message_stream.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/response_message_stream.py rename to libraries/botframework-streaming/botframework/streaming/payloads/response_message_stream.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py b/libraries/botframework-streaming/botframework/streaming/payloads/send_operations.py similarity index 91% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py rename to libraries/botframework-streaming/botframework/streaming/payloads/send_operations.py index 2e81e255d..82a7ecadc 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/send_operations.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/send_operations.py @@ -4,14 +4,14 @@ import asyncio from uuid import UUID -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads.disassemblers import ( +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads.disassemblers import ( CancelDisassembler, RequestDisassembler, ResponseDisassembler, ResponseMessageStreamDisassembler, ) -from botbuilder.streaming.payloads.models import PayloadTypes +from botframework.streaming.payloads.models import PayloadTypes class SendOperations: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py b/libraries/botframework-streaming/botframework/streaming/payloads/stream_manager.py similarity index 92% rename from libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py rename to libraries/botframework-streaming/botframework/streaming/payloads/stream_manager.py index 38bf96f9f..84e4cf4f3 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/payloads/stream_manager.py +++ b/libraries/botframework-streaming/botframework/streaming/payloads/stream_manager.py @@ -4,8 +4,8 @@ from uuid import UUID from typing import Callable, Dict, List -from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler -from botbuilder.streaming.payloads.models import Header +from botframework.streaming.payloads.assemblers import PayloadStreamAssembler +from botframework.streaming.payloads.models import Header class StreamManager: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py b/libraries/botframework-streaming/botframework/streaming/protocol_adapter.py similarity index 93% rename from libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py rename to libraries/botframework-streaming/botframework/streaming/protocol_adapter.py index 59008e32e..71661bdf2 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/protocol_adapter.py +++ b/libraries/botframework-streaming/botframework/streaming/protocol_adapter.py @@ -3,14 +3,14 @@ import asyncio from uuid import UUID, uuid4 -from botbuilder.streaming.payloads import ( +from botframework.streaming.payloads import ( PayloadAssemblerManager, RequestManager, SendOperations, StreamManager, ) -from botbuilder.streaming.payloads.assemblers import PayloadStreamAssembler -from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver +from botframework.streaming.payloads.assemblers import PayloadStreamAssembler +from botframework.streaming.payload_transport import PayloadSender, PayloadReceiver from .receive_request import ReceiveRequest from .receive_response import ReceiveResponse diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py b/libraries/botframework-streaming/botframework/streaming/receive_request.py similarity index 93% rename from libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py rename to libraries/botframework-streaming/botframework/streaming/receive_request.py index c6316edfd..973630bd0 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_request.py +++ b/libraries/botframework-streaming/botframework/streaming/receive_request.py @@ -3,7 +3,7 @@ from typing import List -from botbuilder.streaming.payloads import ContentStream +from botframework.streaming.payloads import ContentStream class ReceiveRequest: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py b/libraries/botframework-streaming/botframework/streaming/receive_response.py similarity index 92% rename from libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py rename to libraries/botframework-streaming/botframework/streaming/receive_response.py index 1da08ee47..517874b5c 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/receive_response.py +++ b/libraries/botframework-streaming/botframework/streaming/receive_response.py @@ -4,8 +4,8 @@ from typing import List, Union, Type from msrest.serialization import Model -from botbuilder.streaming.payloads import ContentStream -from botbuilder.streaming.payloads.models import Serializable +from botframework.streaming.payloads import ContentStream +from botframework.streaming.payloads.models import Serializable class ReceiveResponse: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py b/libraries/botframework-streaming/botframework/streaming/request_handler.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/request_handler.py rename to libraries/botframework-streaming/botframework/streaming/request_handler.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py b/libraries/botframework-streaming/botframework/streaming/streaming_request.py similarity index 94% rename from libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py rename to libraries/botframework-streaming/botframework/streaming/streaming_request.py index 2e4611d88..6157a04d6 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_request.py +++ b/libraries/botframework-streaming/botframework/streaming/streaming_request.py @@ -6,8 +6,8 @@ from typing import List, Union from msrest.serialization import Model -from botbuilder.streaming.payloads import ResponseMessageStream -from botbuilder.streaming.payloads.models import Serializable +from botframework.streaming.payloads import ResponseMessageStream +from botframework.streaming.payloads.models import Serializable class StreamingRequest: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py b/libraries/botframework-streaming/botframework/streaming/streaming_response.py similarity index 91% rename from libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py rename to libraries/botframework-streaming/botframework/streaming/streaming_response.py index f052af399..a97dad475 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/streaming_response.py +++ b/libraries/botframework-streaming/botframework/streaming/streaming_response.py @@ -6,8 +6,8 @@ from typing import List, Union from msrest.serialization import Model -from botbuilder.streaming.payloads import ResponseMessageStream -from botbuilder.streaming.payloads.models import Serializable +from botframework.streaming.payloads import ResponseMessageStream +from botframework.streaming.payloads.models import Serializable class StreamingResponse: diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py b/libraries/botframework-streaming/botframework/streaming/transport/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/__init__.py rename to libraries/botframework-streaming/botframework/streaming/transport/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py b/libraries/botframework-streaming/botframework/streaming/transport/disconnected_event_args.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/disconnected_event_args.py rename to libraries/botframework-streaming/botframework/streaming/transport/disconnected_event_args.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py b/libraries/botframework-streaming/botframework/streaming/transport/streaming_transport_service.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/streaming_transport_service.py rename to libraries/botframework-streaming/botframework/streaming/transport/streaming_transport_service.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py b/libraries/botframework-streaming/botframework/streaming/transport/transport_base.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_base.py rename to libraries/botframework-streaming/botframework/streaming/transport/transport_base.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py b/libraries/botframework-streaming/botframework/streaming/transport/transport_constants.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_constants.py rename to libraries/botframework-streaming/botframework/streaming/transport/transport_constants.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py b/libraries/botframework-streaming/botframework/streaming/transport/transport_receiver_base.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_receiver_base.py rename to libraries/botframework-streaming/botframework/streaming/transport/transport_receiver_base.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py b/libraries/botframework-streaming/botframework/streaming/transport/transport_sender_base.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/transport_sender_base.py rename to libraries/botframework-streaming/botframework/streaming/transport/transport_sender_base.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/__init__.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/__init__.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/__init__.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_close_status.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_close_status.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_close_status.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_message_type.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_message_type.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_message_type.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_server.py similarity index 92% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_server.py index cad838343..67d0d8336 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_server.py +++ b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_server.py @@ -4,15 +4,15 @@ from asyncio import Future, iscoroutinefunction, isfuture from typing import Callable -from botbuilder.streaming import ( +from botframework.streaming import ( ProtocolAdapter, ReceiveResponse, RequestHandler, StreamingRequest, ) -from botbuilder.streaming.payloads import RequestManager -from botbuilder.streaming.payload_transport import PayloadSender, PayloadReceiver -from botbuilder.streaming.transport import DisconnectedEventArgs +from botframework.streaming.payloads import RequestManager +from botframework.streaming.payload_transport import PayloadSender, PayloadReceiver +from botframework.streaming.transport import DisconnectedEventArgs from .web_socket import WebSocket from .web_socket_transport import WebSocketTransport diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_state.py similarity index 100% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_state.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_state.py diff --git a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_transport.py similarity index 97% rename from libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py rename to libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_transport.py index 600e215a4..bd327affa 100644 --- a/libraries/botbuilder-streaming/botbuilder/streaming/transport/web_socket/web_socket_transport.py +++ b/libraries/botframework-streaming/botframework/streaming/transport/web_socket/web_socket_transport.py @@ -4,7 +4,7 @@ import traceback from typing import List -from botbuilder.streaming.transport import TransportReceiverBase, TransportSenderBase +from botframework.streaming.transport import TransportReceiverBase, TransportSenderBase from .web_socket import WebSocket from .web_socket_message_type import WebSocketMessageType diff --git a/libraries/botbuilder-streaming/requirements.txt b/libraries/botframework-streaming/requirements.txt similarity index 100% rename from libraries/botbuilder-streaming/requirements.txt rename to libraries/botframework-streaming/requirements.txt diff --git a/libraries/botbuilder-streaming/setup.py b/libraries/botframework-streaming/setup.py similarity index 74% rename from libraries/botbuilder-streaming/setup.py rename to libraries/botframework-streaming/setup.py index 6f3c2c4cb..c7baf511f 100644 --- a/libraries/botbuilder-streaming/setup.py +++ b/libraries/botframework-streaming/setup.py @@ -13,7 +13,7 @@ root = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(root, "botbuilder", "streaming", "about.py")) as f: +with open(os.path.join(root, "botframework", "streaming", "about.py")) as f: package_info = {} info = f.read() exec(info, package_info) @@ -27,17 +27,17 @@ url=package_info["__uri__"], author=package_info["__author__"], description=package_info["__description__"], - keywords=["BotBuilderStreaming", "bots", "ai", "botframework", "botbuilder",], + keywords=["BotFrameworkStreaming", "bots", "ai", "botframework", "botframework",], long_description=long_description, long_description_content_type="text/x-rst", license=package_info["__license__"], packages=[ - "botbuilder.streaming", - "botbuilder.streaming.payloads", - "botbuilder.streaming.payloads.models", - "botbuilder.streaming.payload_transport", - "botbuilder.streaming.transport", - "botbuilder.streaming.transport.web_socket", + "botframework.streaming", + "botframework.streaming.payloads", + "botframework.streaming.payloads.models", + "botframework.streaming.payload_transport", + "botframework.streaming.transport", + "botframework.streaming.transport.web_socket", ], install_requires=REQUIRES, classifiers=[ diff --git a/libraries/botbuilder-streaming/tests/test_payload_processor.py b/libraries/botframework-streaming/tests/test_payload_processor.py similarity index 62% rename from libraries/botbuilder-streaming/tests/test_payload_processor.py rename to libraries/botframework-streaming/tests/test_payload_processor.py index 0f43bcbd6..cb892ff16 100644 --- a/libraries/botbuilder-streaming/tests/test_payload_processor.py +++ b/libraries/botframework-streaming/tests/test_payload_processor.py @@ -2,10 +2,17 @@ from uuid import UUID, uuid4 import aiounittest -from botbuilder.streaming import ReceiveRequest -from botbuilder.streaming.payloads import StreamManager -from botbuilder.streaming.payloads.assemblers import ReceiveRequestAssembler, PayloadStreamAssembler -from botbuilder.streaming.payloads.models import Header, RequestPayload, StreamDescription +from botframework.streaming import ReceiveRequest +from botframework.streaming.payloads import StreamManager +from botframework.streaming.payloads.assemblers import ( + ReceiveRequestAssembler, + PayloadStreamAssembler, +) +from botframework.streaming.payloads.models import ( + Header, + RequestPayload, + StreamDescription, +) class MockStreamManager(StreamManager): @@ -34,12 +41,18 @@ async def mock_on_completed(identifier: UUID, request: ReceiveRequest): assert len(request.streams) == 1 on_completed_called = True - sut = ReceiveRequestAssembler(header, stream_manager, on_completed=mock_on_completed) + sut = ReceiveRequestAssembler( + header, stream_manager, on_completed=mock_on_completed + ) # Act stream_id: UUID = uuid4() - streams: List[StreamDescription] = [StreamDescription(id=str(stream_id), content_type="json", length=100)] - payload = RequestPayload(verb="POST", path="/api/messages", streams=streams).to_json() + streams: List[StreamDescription] = [ + StreamDescription(id=str(stream_id), content_type="json", length=100) + ] + payload = RequestPayload( + verb="POST", path="/api/messages", streams=streams + ).to_json() payload_stream: List[int] = list(bytes(payload, "utf-8")) await sut.process_request(payload_stream) diff --git a/libraries/botbuilder-streaming/tests/test_payload_receiver.py b/libraries/botframework-streaming/tests/test_payload_receiver.py similarity index 91% rename from libraries/botbuilder-streaming/tests/test_payload_receiver.py rename to libraries/botframework-streaming/tests/test_payload_receiver.py index 315bd23a7..c9e2aa58c 100644 --- a/libraries/botbuilder-streaming/tests/test_payload_receiver.py +++ b/libraries/botframework-streaming/tests/test_payload_receiver.py @@ -2,9 +2,9 @@ import aiounittest -from botbuilder.streaming import PayloadStream -from botbuilder.streaming.payload_transport import PayloadReceiver -from botbuilder.streaming.transport import TransportReceiverBase +from botframework.streaming import PayloadStream +from botframework.streaming.payload_transport import PayloadReceiver +from botframework.streaming.transport import TransportReceiverBase class MockTransportReceiver(TransportReceiverBase): diff --git a/libraries/botbuilder-streaming/tests/test_payload_sender.py b/libraries/botframework-streaming/tests/test_payload_sender.py similarity index 86% rename from libraries/botbuilder-streaming/tests/test_payload_sender.py rename to libraries/botframework-streaming/tests/test_payload_sender.py index 5aaeb880d..242e0de45 100644 --- a/libraries/botbuilder-streaming/tests/test_payload_sender.py +++ b/libraries/botframework-streaming/tests/test_payload_sender.py @@ -3,10 +3,10 @@ from uuid import UUID, uuid4 import aiounittest -from botbuilder.streaming.payload_transport import PayloadSender -from botbuilder.streaming.payloads import HeaderSerializer -from botbuilder.streaming.payloads.models import Header -from botbuilder.streaming.transport import TransportSenderBase +from botframework.streaming.payload_transport import PayloadSender +from botframework.streaming.payloads import HeaderSerializer +from botframework.streaming.payloads.models import Header +from botframework.streaming.transport import TransportSenderBase class MockTransportSender(TransportSenderBase): From 456135247f6462b2d119d6eb27ccb2645f6e2939 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 6 May 2021 11:04:31 -0700 Subject: [PATCH 35/37] Updating pipeline --- pipelines/botbuilder-python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/botbuilder-python-ci.yml b/pipelines/botbuilder-python-ci.yml index 221f1a1c3..af2fc58c6 100644 --- a/pipelines/botbuilder-python-ci.yml +++ b/pipelines/botbuilder-python-ci.yml @@ -48,7 +48,7 @@ jobs: pip install -e ./libraries/botbuilder-integration-applicationinsights-aiohttp pip install -e ./libraries/botbuilder-adapters-slack pip install -e ./libraries/botbuilder-integration-aiohttp - pip install -e ./libraries/botbuilder-streaming + pip install -e ./libraries/botframework-streaming pip install -r ./libraries/botframework-connector/tests/requirements.txt pip install -r ./libraries/botbuilder-core/tests/requirements.txt pip install -r ./libraries/botbuilder-ai/tests/requirements.txt From 65aca0b61c83487293225363d531e726406ca48c Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 6 May 2021 11:25:21 -0700 Subject: [PATCH 36/37] Removing sleep() safety measure --- .../botframework/streaming/payload_transport/send_queue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py index 4163e7558..67501502a 100644 --- a/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py @@ -27,7 +27,6 @@ async def _process(self): while True: try: while True: - await sleep(0.2) item = await self._queue.get() try: await self._action(item) From 4da1fc26f819f5cef7579121633103be92302232 Mon Sep 17 00:00:00 2001 From: Axel Suarez Date: Thu, 6 May 2021 12:02:08 -0700 Subject: [PATCH 37/37] Remove unused import --- .../botframework/streaming/payload_transport/send_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py index 67501502a..d337d911a 100644 --- a/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py +++ b/libraries/botframework-streaming/botframework/streaming/payload_transport/send_queue.py @@ -3,7 +3,7 @@ import traceback -from asyncio import Queue, ensure_future, sleep +from asyncio import Queue, ensure_future from typing import Awaitable, Callable