From 2aaf3d2e2277e4ed3d2bbf17fd13d14ec45694c8 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Fri, 22 Nov 2019 13:17:17 -0800 Subject: [PATCH 01/11] - added cosmosdb_partitioned_storage - added storage_base_tests - added test_cosmos_partitioned_storage --- .../tests/test_cosmos_partitioned_storage.py | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py new file mode 100644 index 000000000..f7515ffca --- /dev/null +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -0,0 +1,206 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from unittest.mock import Mock +import azure.cosmos.errors as cosmos_errors +from azure.cosmos import documents +from azure.cosmos.cosmos_client import CosmosClient +import pytest +from botbuilder.core import StoreItem +from botbuilder.azure import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig +from tests import StorageBaseTests + +EMULATOR_RUNNING = True + + +def get_settings() -> CosmosDbPartitionedConfig: + return CosmosDbPartitionedConfig( + cosmos_db_endpoint="https://localhost:8081", + auth_key="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + database_id="test-db", + container_id="bot-storage" + ) + + +def get_storage(): + return CosmosDbPartitionedStorage(get_settings()) + + +async def reset(): + storage = CosmosDbPartitionedStorage(get_settings()) + await storage.initialize() + try: + storage.client.DeleteDatabase(database_link="dbs/" + get_settings().database_id) + except cosmos_errors.HTTPFailure: + pass + + +def get_mock_client(identifier: str = "1"): + # pylint: disable=attribute-defined-outside-init, invalid-name + mock = MockClient() + + mock.QueryDatabases = Mock(return_value=[]) + mock.QueryContainers = Mock(return_value=[]) + mock.CreateDatabase = Mock(return_value={"id": identifier}) + mock.CreateContainer = Mock(return_value={"id": identifier}) + + return mock + + +class MockClient(CosmosClient): + def __init__(self): # pylint: disable=super-init-not-called + pass + + +class SimpleStoreItem(StoreItem): + def __init__(self, counter=1, e_tag="*"): + super(SimpleStoreItem, self).__init__() + self.counter = counter + self.e_tag = e_tag + + +class TestCosmosDbStorageConstructor: + @pytest.mark.asyncio + async def test_raises_error_when_instantiated_with_no_arguments(self): + try: + # noinspection PyArgumentList + CosmosDbPartitionedStorage() + except Exception as e: + assert e + + @pytest.mark.asyncio + async def test_raises_error_when_no_endpoint_provided(self): + no_endpoint = get_settings() + no_endpoint.cosmos_db_endpoint = None + try: + CosmosDbPartitionedStorage(no_endpoint) + except Exception as e: + assert e + + @pytest.mark.asyncio + async def test_raises_error_when_no_auth_key_provided(self): + no_auth_key = get_settings() + no_auth_key.auth_key = None + try: + CosmosDbPartitionedStorage(no_auth_key) + except Exception as e: + assert e + + @pytest.mark.asyncio + async def test_raises_error_when_no_database_id_provided(self): + no_database_id = get_settings() + no_database_id.database_id = None + try: + CosmosDbPartitionedStorage(no_database_id) + except Exception as e: + assert e + + @pytest.mark.asyncio + async def test_raises_error_when_no_container_id_provided(self): + no_container_id = get_settings() + no_container_id.container_id = None + try: + CosmosDbPartitionedStorage(no_container_id) + except Exception as e: + assert e + + @pytest.mark.asyncio + async def test_passes_cosmos_client_options(self): + settings_with_options = get_settings() + + connection_policy = documents.ConnectionPolicy() + connection_policy.DisableSSLVerification = True + + settings_with_options.cosmos_client_options = { + "connection_policy": connection_policy, + "consistency_level": documents.ConsistencyLevel.Eventual + } + + client = CosmosDbPartitionedStorage(settings_with_options) + await client.initialize() + + assert client.client.connection_policy.DisableSSLVerification is True + assert client.client.default_headers['x-ms-consistency-level'] == documents.ConsistencyLevel.Eventual + + +class TestCosmosDbStorageBaseStorageTests: + @pytest.mark.asyncio + async def test_return_empty_object_when_reading_unknown_key(self): + await reset() + + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_writing(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_does_not_raise_when_writing_no_items(self): + await reset() + + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_create_object(self): + await reset() + + test_ran = await StorageBaseTests.create_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_crazy_keys(self): + await reset() + + test_ran = await StorageBaseTests.handle_crazy_keys(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_update_object(self): + await reset() + + test_ran = await StorageBaseTests.update_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_delete_object(self): + await reset() + + test_ran = await StorageBaseTests.delete_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_perform_batch_operations(self): + await reset() + + test_ran = await StorageBaseTests.perform_batch_operations(get_storage()) + + assert test_ran + + # TODO: Re-enable after the dialog_stack PR gets merged + # @pytest.mark.asyncio + # async def test_proceeds_through_waterfall(self): + # await reset() + # + # test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) + # + # assert test_ran From e8392cdfe471ffc4c3e1f4137f9b710adddd56c8 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Fri, 22 Nov 2019 13:26:39 -0800 Subject: [PATCH 02/11] - added cosmosdb_partitioned_storage - added storage_base_tests - added test_cosmos_partitioned_storage --- .../botbuilder/azure/__init__.py | 3 + .../azure/cosmosdb_partitioned_storage.py | 280 ++++++++++++++++ .../tests/test_cosmos_partitioned_storage.py | 4 +- libraries/botbuilder-core/tests/__init__.py | 12 + .../tests/storage_base_tests.py | 302 ++++++++++++++++++ 5 files changed, 599 insertions(+), 2 deletions(-) create mode 100644 libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py create mode 100644 libraries/botbuilder-core/tests/__init__.py create mode 100644 libraries/botbuilder-core/tests/storage_base_tests.py diff --git a/libraries/botbuilder-azure/botbuilder/azure/__init__.py b/libraries/botbuilder-azure/botbuilder/azure/__init__.py index 54dea209d..703fc465c 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/__init__.py +++ b/libraries/botbuilder-azure/botbuilder/azure/__init__.py @@ -7,6 +7,7 @@ from .about import __version__ from .cosmosdb_storage import CosmosDbStorage, CosmosDbConfig, CosmosDbKeyEscape +from .cosmosdb_partitioned_storage import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig from .blob_storage import BlobStorage, BlobStorageSettings __all__ = [ @@ -15,5 +16,7 @@ "CosmosDbStorage", "CosmosDbConfig", "CosmosDbKeyEscape", + "CosmosDbPartitionedStorage", + "CosmosDbPartitionedConfig", "__version__", ] diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py new file mode 100644 index 000000000..c9c725b24 --- /dev/null +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py @@ -0,0 +1,280 @@ +"""CosmosDB Middleware for Python Bot Framework. + +This is middleware to store items in CosmosDB. +Part of the Azure Bot Framework in Python. +""" + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +from hashlib import sha256 +from typing import Dict, List +from threading import Semaphore +import json + +from azure.cosmos import documents, http_constants +from jsonpickle.pickler import Pickler +from jsonpickle.unpickler import Unpickler +import azure.cosmos.cosmos_client as cosmos_client # pylint: disable=no-name-in-module,import-error +import azure.cosmos.errors as cosmos_errors # pylint: disable=no-name-in-module,import-error +from botbuilder.core.storage import Storage +from botbuilder.azure import CosmosDbKeyEscape + + +class CosmosDbPartitionedConfig: + """The class for partitioned CosmosDB configuration for the Azure Bot Framework.""" + + def __init__( + self, + cosmos_db_endpoint: str = None, + auth_key: str = None, + database_id: str = None, + container_id: str = None, + cosmos_client_options: dict = None, + container_throughput: int = None, + **kwargs, + ): + """Create the Config object. + + :param cosmos_db_endpoint: The CosmosDB endpoint. + :param auth_key: The authentication key for Cosmos DB. + :param database_id: The database identifier for Cosmos DB instance. + :param container_id: The container identifier. + :param cosmos_client_options: The options for the CosmosClient. Currently only supports connection_policy and + consistency_level + :param container_throughput: The throughput set when creating the Container. Defaults to 400. + :return CosmosDbPartitionedConfig: + """ + self.__config_file = kwargs.get("filename") + if self.__config_file: + kwargs = json.load(open(self.__config_file)) + self.cosmos_db_endpoint = cosmos_db_endpoint or kwargs.get("cosmos_db_endpoint") + self.auth_key = auth_key or kwargs.get("auth_key") + self.database_id = database_id or kwargs.get("database_id") + self.container_id = container_id or kwargs.get("container_id") + self.cosmos_client_options = cosmos_client_options or kwargs.get( + "cosmos_client_options", {} + ) + self.container_throughput = container_throughput or kwargs.get( + "container_throughput" + ) + + +class CosmosDbPartitionedStorage(Storage): + """The class for partitioned CosmosDB middleware for the Azure Bot Framework.""" + + def __init__( + self, + config: CosmosDbPartitionedConfig + ): + """Create the storage object. + + :param config: + """ + super(CosmosDbPartitionedStorage, self).__init__() + self.config = config + self.client = None + self.database = None + self.container = None + self.__semaphore = Semaphore() + + async def read(self, keys: List[str]) -> Dict[str, object]: + """Read storeitems from storage. + + :param keys: + :return dict: + """ + if not keys: + raise Exception("Keys are required when reading") + + await self.initialize() + + store_items = {} + + for key in keys: + try: + escaped_key = CosmosDbKeyEscape.sanitize_key(key) + + read_item_response = self.client.ReadItem( + self.__item_link(escaped_key), {"partitionKey": escaped_key} + ) + document_store_item = read_item_response + if document_store_item: + store_items[document_store_item["realId"]] = document_store_item["document"] + store_items[document_store_item["realId"]]["e_tag"] = document_store_item["_etag"] + # When an item is not found a CosmosException is thrown, but we want to + # return an empty collection so in this instance we catch and do not rethrow. + # Throw for any other exception. + except cosmos_errors.HTTPFailure as err: + if err.status_code == cosmos_errors.http_constants.StatusCodes.NOT_FOUND: + continue + else: + raise err + except Exception as err: + raise err + return store_items + + async def write(self, changes: Dict[str, object]): + """Save storeitems to storage. + + :param changes: + :return: + """ + if changes is None: + raise Exception("Changes are required when writing") + elif not changes: + return + + await self.initialize() + + for (key, change) in changes.items(): + e_tag = change.get("e_tag", None) + doc = { + "id": CosmosDbKeyEscape.sanitize_key(key), + "realId": key, + "document": self.__create_dict(change), + } + if e_tag == "": + raise Exception("cosmosdb_storage.write(): etag missing") + + accessCondition = {"accessCondition": {"type": "IfMatch", "condition": e_tag}} + options = accessCondition if e_tag != "*" and e_tag and e_tag != "" else None + try: + self.client.UpsertItem( + database_or_Container_link=self.__container_link, + document=doc, + options=options, + ) + except cosmos_errors.HTTPFailure as err: + raise err + except Exception as err: + raise err + + async def delete(self, keys: List[str]): + """Remove storeitems from storage. + + :param keys: + :return: + """ + await self.initialize() + + for key in keys: + escaped_key = CosmosDbKeyEscape.sanitize_key(key) + try: + self.client.DeleteItem( + document_link=self.__item_link(escaped_key), + options={"partitionKey": escaped_key}, + ) + except cosmos_errors.HTTPFailure as err: + if err.status_code == cosmos_errors.http_constants.StatusCodes.NOT_FOUND: + continue + else: + raise err + except Exception as err: + raise err + + @staticmethod + def __create_si(result) -> object: + """Create an object from a result out of CosmosDB. + + :param result: + :return object: + """ + # get the document item from the result and turn into a dict + doc = result.get("document") + # read the e_tag from Cosmos + if result.get("_etag"): + doc["e_tag"] = result["_etag"] + + result_obj = Unpickler().restore(doc) + + # create and return the object + return result_obj + + @staticmethod + def __create_dict(store_item: object) -> Dict: + """Return the dict of an object. + + This eliminates non_magic attributes and the e_tag. + + :param store_item: + :return dict: + """ + # read the content + json_dict = Pickler().flatten(store_item) + if "e_tag" in json_dict: + del json_dict["e_tag"] + + # loop through attributes and write and return a dict + return json_dict + + async def initialize(self): + if not self.container: + if not self.client: + self.client = cosmos_client.CosmosClient( + self.config.cosmos_db_endpoint, + {"masterKey": self.config.auth_key}, + self.config.cosmos_client_options.get("connection_policy", None), + self.config.cosmos_client_options.get("consistency_level", None), + ) + + if not self.database: + with self.__semaphore: + try: + self.database = self.client.CreateDatabase( + {"id": self.config.database_id} + ) + except cosmos_errors.HTTPFailure: + self.database = self.client.ReadDatabase( + "dbs/" + self.config.database_id + ) + + if not self.container: + with self.__semaphore: + container_def = { + "id": self.config.container_id, + "partitionKey": { + "paths": ["/id"], + "kind": documents.PartitionKind.Hash, + }, + } + try: + self.container = self.client.CreateContainer( + "dbs/" + self.database["id"], + container_def, + {"offerThroughput": 400}, + ) + except cosmos_errors.HTTPFailure as err: + if err.status_code == http_constants.StatusCodes.CONFLICT: + self.container = self.client.ReadContainer( + "dbs/" + + self.database["id"] + + "/colls/" + + container_def["id"] + ) + else: + raise err + + def __item_link(self, identifier) -> str: + """Return the item link of a item in the container. + + :param identifier: + :return str: + """ + return self.__container_link + "/docs/" + identifier + + @property + def __container_link(self) -> str: + """Return the container link in the database. + + :param: + :return str: + """ + return self.__database_link + "/colls/" + self.config.container_id + + @property + def __database_link(self) -> str: + """Return the database link. + + :return str: + """ + return "dbs/" + self.config.database_id diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index f7515ffca..dc2f3d264 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -59,7 +59,7 @@ def __init__(self, counter=1, e_tag="*"): self.e_tag = e_tag -class TestCosmosDbStorageConstructor: +class TestCosmosDbPartitionedStorageConstructor: @pytest.mark.asyncio async def test_raises_error_when_instantiated_with_no_arguments(self): try: @@ -123,7 +123,7 @@ async def test_passes_cosmos_client_options(self): assert client.client.default_headers['x-ms-consistency-level'] == documents.ConsistencyLevel.Eventual -class TestCosmosDbStorageBaseStorageTests: +class TestCosmosDbPartitionedStorageBaseStorageTests: @pytest.mark.asyncio async def test_return_empty_object_when_reading_unknown_key(self): await reset() diff --git a/libraries/botbuilder-core/tests/__init__.py b/libraries/botbuilder-core/tests/__init__.py new file mode 100644 index 000000000..2bef95eb6 --- /dev/null +++ b/libraries/botbuilder-core/tests/__init__.py @@ -0,0 +1,12 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from .storage_base_tests import StorageBaseTests + +__all__ = [ + "StorageBaseTests" +] diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-core/tests/storage_base_tests.py new file mode 100644 index 000000000..590e5296c --- /dev/null +++ b/libraries/botbuilder-core/tests/storage_base_tests.py @@ -0,0 +1,302 @@ +""" +Base tests that all storage providers should implement in their own tests. +They handle the storage-based assertions, internally. + +All tests return true if assertions pass to indicate that the code ran to completion, passing internal assertions. +Therefore, all tests using theses static tests should strictly check that the method returns true. + +:Example: +TODO: write example +""" +import pytest +from botbuilder.core import ConversationState, TurnContext, MessageFactory +from botbuilder.core.adapters import TestAdapter +from botbuilder.dialogs import DialogSet, DialogTurnStatus, TextPrompt, PromptValidatorContext, WaterfallStepContext, \ + Dialog, WaterfallDialog + + +class StorageBaseTests: + @staticmethod + async def return_empty_object_when_reading_unknown_key(storage) -> bool: + result = await storage.read(['unknown']) + + assert result is not None + assert len(result) == 0 + + return True + + @staticmethod + async def handle_null_keys_when_reading(storage) -> bool: + with pytest.raises(Exception) as err: + await storage.read(None) + assert err.value.args[0] == "Keys are required when reading" + + return True + + @staticmethod + async def handle_null_keys_when_writing(storage) -> bool: + with pytest.raises(Exception) as err: + await storage.write(None) + assert err.value.args[0] == "Changes are required when writing" + + return True + + @staticmethod + async def does_not_raise_when_writing_no_items(storage) -> bool: + # noinspection PyBroadException + try: + await storage.write([]) + except: + pytest.fail("Should not raise") + + return True + + @staticmethod + async def create_object(storage) -> bool: + store_items = { + "createPoco": {"id": 1}, + "createPocoStoreItem": {"id": 2}, + } + + await storage.write(store_items) + + read_store_items = await storage.read(store_items.keys()) + + assert store_items["createPoco"]["id"] == read_store_items["createPoco"]["id"] + assert store_items["createPocoStoreItem"]["id"] == read_store_items["createPocoStoreItem"]["id"] + assert read_store_items["createPoco"]["e_tag"] is not None + assert read_store_items["createPocoStoreItem"]["e_tag"] is not None + + return True + + @staticmethod + async def handle_crazy_keys(storage) -> bool: + key = '!@#$%^&*()~/\\><,.?\';"\`~' + store_item = {"id": 1} + store_items = {key: store_item} + + await storage.write(store_items) + + read_store_items = await storage.read(store_items.keys()) + + assert read_store_items[key] is not None + assert read_store_items[key]["id"] == 1 + + return True + + @staticmethod + async def update_object(storage) -> bool: + original_store_items = { + "pocoItem": {"id": 1, "count": 1}, + "pocoStoreItem": {"id": 1, "count": 1} + } + + # 1st write should work + await storage.write(original_store_items) + + loaded_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) + + update_poco_item = loaded_store_items["pocoItem"] + update_poco_item["e_tag"] = None + update_poco_store_item = loaded_store_items["pocoStoreItem"] + assert update_poco_store_item["e_tag"] is not None + + # 2nd write should work + update_poco_item["count"] += 1 + update_poco_store_item["count"] += 1 + + await storage.write(loaded_store_items) + + reloaded_store_items = await storage.read(loaded_store_items.keys()) + + reloaded_update_poco_item = reloaded_store_items["pocoItem"] + reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] + + assert reloaded_update_poco_item["e_tag"] is not None + assert update_poco_store_item["e_tag"] != reloaded_update_poco_store_item["e_tag"] + assert reloaded_update_poco_item["count"] == 2 + assert reloaded_update_poco_store_item["count"] == 2 + + # Write with old e_tag should succeed for non-storeItem + update_poco_item["count"] = 123 + await storage.write({"pocoItem": update_poco_item}) + + # Write with old eTag should FAIL for storeItem + update_poco_store_item["count"] = 123 + + with pytest.raises(Exception) as err: + await storage.write({"pocoStoreItem": update_poco_store_item}) + assert err.value is not None + + reloaded_store_items2 = await storage.read(["pocoItem", "pocoStoreItem"]) + + reloaded_poco_item2 = reloaded_store_items2["pocoItem"] + reloaded_poco_item2["e_tag"] = None + reloaded_poco_store_item2 = reloaded_store_items2["pocoStoreItem"] + + assert reloaded_poco_item2["count"] == 123 + assert reloaded_poco_store_item2["count"] == 2 + + # write with wildcard etag should work + reloaded_poco_item2["count"] = 100 + reloaded_poco_store_item2["count"] = 100 + reloaded_poco_store_item2["e_tag"] = "*" + + wildcard_etag_dict = { + "pocoItem": reloaded_poco_item2, + "pocoStoreItem": reloaded_poco_store_item2 + } + + await storage.write(wildcard_etag_dict) + + reloaded_store_items3 = await storage.read(["pocoItem", "pocoStoreItem"]) + + assert reloaded_store_items3["pocoItem"]["count"] == 100 + assert reloaded_store_items3["pocoStoreItem"]["count"] == 100 + + # Write with empty etag should not work + reloaded_store_items4 = await storage.read(["pocoStoreItem"]) + reloaded_store_item4 = reloaded_store_items4["pocoStoreItem"] + + assert reloaded_store_item4 is not None + + reloaded_store_item4["e_tag"] = "" + dict2 = {"pocoStoreItem": reloaded_store_item4} + + with pytest.raises(Exception) as err: + await storage.write(dict2) + assert err.value is not None + + final_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) + assert final_store_items["pocoItem"]["count"] == 100 + assert final_store_items["pocoStoreItem"]["count"] == 100 + + return True + + @staticmethod + async def delete_object(storage) -> bool: + store_items = { + "delete1": {"id": 1, "count": 1} + } + + await storage.write(store_items) + + read_store_items = await storage.read(["delete1"]) + + assert read_store_items["delete1"]["e_tag"] + assert read_store_items["delete1"]["count"] == 1 + + await storage.delete(["delete1"]) + + reloaded_store_items = await storage.read(["delete1"]) + + assert reloaded_store_items.get("delete1", None) is None + + return True + + @staticmethod + async def delete_unknown_object(storage) -> bool: + # noinspection PyBroadException + try: + await storage.delete(["unknown_key"]) + except: + pytest.fail("Should not raise") + + return True + + @staticmethod + async def perform_batch_operations(storage) -> bool: + await storage.write({ + "batch1": {"count": 10}, + "batch2": {"count": 20}, + "batch3": {"count": 30}, + }) + + result = await storage.read(["batch1", "batch2", "batch3"]) + + assert result.get("batch1", None) is not None + assert result.get("batch2", None) is not None + assert result.get("batch3", None) is not None + assert result["batch1"]["count"] == 10 + assert result["batch2"]["count"] == 20 + assert result["batch3"]["count"] == 30 + assert result["batch1"].get("e_tag", None) is not None + assert result["batch2"].get("e_tag", None) is not None + assert result["batch3"].get("e_tag", None) is not None + + await storage.delete(["batch1", "batch2", "batch3"]) + + result = await storage.read(["batch1", "batch2", "batch3"]) + + assert result.get("batch1", None) is None + assert result.get("batch2", None) is None + assert result.get("batch3", None) is None + + return True + +# TODO: Re-enable after the dialog_stack PR gets merged + # @staticmethod + # async def proceeds_through_waterfall(storage) -> bool: + # convo_state = ConversationState(storage) + # + # dialog_state = convo_state.create_property("dialog_state") + # dialogs = DialogSet(dialog_state) + # + # async def exec_test(turn_context: TurnContext) -> None: + # dialog_context = await dialogs.create_context(turn_context) + # results = await dialog_context.continue_dialog() + # if results.status == DialogTurnStatus.Empty: + # await dialog_context.begin_dialog("waterfall_dialog") + # else: + # if results.status == DialogTurnStatus.Complete: + # await turn_context.send_activity(results.result) + # await convo_state.save_changes(turn_context) + # + # adapter = TestAdapter(exec_test) + # + # async def prompt_validator(prompt_context: PromptValidatorContext) -> bool: + # result = prompt_context.recognized.value + # if len(result) > 3: + # succeeded_message = MessageFactory.text(f"You got it at the {prompt_context.attempt_count}th try!") + # await prompt_context.context.send_activity(succeeded_message) + # return True + # + # reply = MessageFactory.text(f"Please send a name that is longer than 3 characters. {prompt_context.attempt_count}") + # await prompt_context.context.send_activity(reply) + # return False + # + # dialogs.add(TextPrompt("text_prompt", prompt_validator)) + # + # async def step_1(step_context: WaterfallStepContext) -> DialogTurnStatus: + # assert isinstance(step_context.active_dialog.state["stepIndex"], int) + # await step_context.context.send_activity("step1") + # return Dialog.end_of_turn + # + # async def step_2(step_context: WaterfallStepContext) -> None: + # assert isinstance(step_context.active_dialog.state["stepIndex"], int) + # await step_context.prompt("text_prompt", {"prompt": MessageFactory.text("Please type your name.")}) + # + # async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: + # assert isinstance(step_context.active_dialog.state["stepIndex"], int) + # await step_context.context.send_activity("step3") + # return Dialog.end_of_turn + # + # steps = [step_1, step_2, step_3] + # + # dialogs.add(WaterfallDialog("waterfall_dialog", steps)) + # + # step1 = await adapter.send("hello") + # step2 = await step1.assert_reply("step1") + # step3 = await step2.send("hello") + # step4 = await step3.assert_reply("Please type your name") + # step5 = await step4.send("hi") + # step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 1") + # step7 = await step6.send("hi") + # step8 = await step7.assert_reply("Please send a name that is longer than 3 characters. 2") + # step9 = await step8.send("hi") + # step10 = await step9.assert_reply("Please send a name that is longer than 3 characters. 3") + # step11 = await step10.send("Kyle") + # step12 = await step11.assert_reply("You got it at the 4th try!") + # await step12.assert_reply("step3") + # + # return True From 66656411e07b2c4d783f2741001a9fd340298b1f Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Tue, 26 Nov 2019 11:37:50 -0800 Subject: [PATCH 03/11] fixed bot_state so dialog tests pass --- .../azure/cosmosdb_partitioned_storage.py | 74 ++++----- .../botbuilder/azure/cosmosdb_storage.py | 2 +- .../tests/test_cosmos_partitioned_storage.py | 15 +- .../tests/test_cosmos_storage.py | 95 +++++++++++- .../botbuilder/core/bot_state.py | 4 +- .../tests/storage_base_tests.py | 143 +++++++++--------- 6 files changed, 216 insertions(+), 117 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py index c9c725b24..8c7f046f4 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py @@ -99,8 +99,7 @@ async def read(self, keys: List[str]) -> Dict[str, object]: ) document_store_item = read_item_response if document_store_item: - store_items[document_store_item["realId"]] = document_store_item["document"] - store_items[document_store_item["realId"]]["e_tag"] = document_store_item["_etag"] + store_items[document_store_item["realId"]] = self.__create_si(document_store_item) # When an item is not found a CosmosException is thrown, but we want to # return an empty collection so in this instance we catch and do not rethrow. # Throw for any other exception. @@ -172,41 +171,6 @@ async def delete(self, keys: List[str]): except Exception as err: raise err - @staticmethod - def __create_si(result) -> object: - """Create an object from a result out of CosmosDB. - - :param result: - :return object: - """ - # get the document item from the result and turn into a dict - doc = result.get("document") - # read the e_tag from Cosmos - if result.get("_etag"): - doc["e_tag"] = result["_etag"] - - result_obj = Unpickler().restore(doc) - - # create and return the object - return result_obj - - @staticmethod - def __create_dict(store_item: object) -> Dict: - """Return the dict of an object. - - This eliminates non_magic attributes and the e_tag. - - :param store_item: - :return dict: - """ - # read the content - json_dict = Pickler().flatten(store_item) - if "e_tag" in json_dict: - del json_dict["e_tag"] - - # loop through attributes and write and return a dict - return json_dict - async def initialize(self): if not self.container: if not self.client: @@ -254,6 +218,42 @@ async def initialize(self): else: raise err + @staticmethod + def __create_si(result) -> object: + """Create an object from a result out of CosmosDB. + + :param result: + :return object: + """ + # get the document item from the result and turn into a dict + doc = result.get("document") + # read the e_tag from Cosmos + if result.get("_etag"): + doc["e_tag"] = result["_etag"] + + result_obj = Unpickler().restore(doc) + + # create and return the object + return result_obj + + @staticmethod + def __create_dict(store_item: object) -> Dict: + """Return the dict of an object. + + This eliminates non_magic attributes and the e_tag. + + :param store_item: + :return dict: + """ + # read the content + json_dict = Pickler().flatten(store_item) + if "e_tag" in json_dict: + del json_dict["e_tag"] + + # loop through attributes and write and return a dict + return json_dict + + def __item_link(self, identifier) -> str: """Return the item link of a item in the container. diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py index c8a25a017..d6f43c850 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py @@ -167,7 +167,7 @@ async def write(self, changes: Dict[str, object]): # iterate over the changes for (key, change) in changes.items(): # store the e_tag - e_tag = change.e_tag + e_tag = change.get("e_tag", None) # create the new document doc = { "id": CosmosDbKeyEscape.sanitize_key(key), diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index dc2f3d264..7b4e017ff 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -196,11 +196,10 @@ async def test_perform_batch_operations(self): assert test_ran - # TODO: Re-enable after the dialog_stack PR gets merged - # @pytest.mark.asyncio - # async def test_proceeds_through_waterfall(self): - # await reset() - # - # test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) - # - # assert test_ran + @pytest.mark.asyncio + async def test_proceeds_through_waterfall(self): + await reset() + + test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) + + assert test_ran diff --git a/libraries/botbuilder-azure/tests/test_cosmos_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_storage.py index a9bfe5191..3f1a7d125 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_storage.py @@ -9,6 +9,8 @@ from botbuilder.azure import CosmosDbStorage, CosmosDbConfig # local cosmosdb emulator instance cosmos_db_config +from tests import StorageBaseTests + COSMOS_DB_CONFIG = CosmosDbConfig( endpoint="https://localhost:8081", masterkey="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", @@ -18,6 +20,10 @@ EMULATOR_RUNNING = False +def get_storage(): + return CosmosDbStorage(COSMOS_DB_CONFIG) + + async def reset(): storage = CosmosDbStorage(COSMOS_DB_CONFIG) try: @@ -50,7 +56,7 @@ def __init__(self, counter=1, e_tag="*"): self.e_tag = e_tag -class TestCosmosDbStorage: +class TestCosmosDbStorageConstructor: @pytest.mark.asyncio async def test_cosmos_storage_init_should_error_without_cosmos_db_config(self): try: @@ -86,6 +92,93 @@ async def test_creation_request_options_era_being_called(self): "dbs/" + test_id, {"id": test_id}, test_config.container_creation_options ) + +class TestCosmosDbStorageBaseStorageTests: + @pytest.mark.asyncio + async def test_return_empty_object_when_reading_unknown_key(self): + await reset() + + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_writing(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_does_not_raise_when_writing_no_items(self): + await reset() + + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_create_object(self): + await reset() + + test_ran = await StorageBaseTests.create_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_crazy_keys(self): + await reset() + + test_ran = await StorageBaseTests.handle_crazy_keys(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_update_object(self): + await reset() + + test_ran = await StorageBaseTests.update_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_delete_object(self): + await reset() + + test_ran = await StorageBaseTests.delete_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_perform_batch_operations(self): + await reset() + + test_ran = await StorageBaseTests.perform_batch_operations(get_storage()) + + assert test_ran + + # TODO: Re-enable after the dialog_stack PR gets merged + @pytest.mark.asyncio + async def test_proceeds_through_waterfall(self): + await reset() + + test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) + + assert test_ran + + +class TestCosmosDbStorage: + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_cosmos_storage_init_should_work_with_just_endpoint_and_key(self): diff --git a/libraries/botbuilder-core/botbuilder/core/bot_state.py b/libraries/botbuilder-core/botbuilder/core/bot_state.py index dc835a9bd..4c9a178b4 100644 --- a/libraries/botbuilder-core/botbuilder/core/bot_state.py +++ b/libraries/botbuilder-core/botbuilder/core/bot_state.py @@ -8,6 +8,7 @@ from .turn_context import TurnContext from .storage import Storage from .property_manager import PropertyManager +from jsonpickle.pickler import Pickler class CachedBotState: @@ -24,8 +25,7 @@ def is_changed(self) -> bool: return self.hash != self.compute_hash(self.state) def compute_hash(self, obj: object) -> str: - # TODO: Should this be compatible with C# JsonConvert.SerializeObject ? - return str(obj) + return str(Pickler().flatten(obj)) class BotState(PropertyManager): diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-core/tests/storage_base_tests.py index 590e5296c..ea1c60776 100644 --- a/libraries/botbuilder-core/tests/storage_base_tests.py +++ b/libraries/botbuilder-core/tests/storage_base_tests.py @@ -9,10 +9,11 @@ TODO: write example """ import pytest -from botbuilder.core import ConversationState, TurnContext, MessageFactory +from botbuilder.azure import BlobStorage, BlobStorageSettings +from botbuilder.core import ConversationState, TurnContext, MessageFactory, MemoryStorage from botbuilder.core.adapters import TestAdapter from botbuilder.dialogs import DialogSet, DialogTurnStatus, TextPrompt, PromptValidatorContext, WaterfallStepContext, \ - Dialog, WaterfallDialog + Dialog, WaterfallDialog, PromptOptions class StorageBaseTests: @@ -234,69 +235,75 @@ async def perform_batch_operations(storage) -> bool: return True -# TODO: Re-enable after the dialog_stack PR gets merged - # @staticmethod - # async def proceeds_through_waterfall(storage) -> bool: - # convo_state = ConversationState(storage) - # - # dialog_state = convo_state.create_property("dialog_state") - # dialogs = DialogSet(dialog_state) - # - # async def exec_test(turn_context: TurnContext) -> None: - # dialog_context = await dialogs.create_context(turn_context) - # results = await dialog_context.continue_dialog() - # if results.status == DialogTurnStatus.Empty: - # await dialog_context.begin_dialog("waterfall_dialog") - # else: - # if results.status == DialogTurnStatus.Complete: - # await turn_context.send_activity(results.result) - # await convo_state.save_changes(turn_context) - # - # adapter = TestAdapter(exec_test) - # - # async def prompt_validator(prompt_context: PromptValidatorContext) -> bool: - # result = prompt_context.recognized.value - # if len(result) > 3: - # succeeded_message = MessageFactory.text(f"You got it at the {prompt_context.attempt_count}th try!") - # await prompt_context.context.send_activity(succeeded_message) - # return True - # - # reply = MessageFactory.text(f"Please send a name that is longer than 3 characters. {prompt_context.attempt_count}") - # await prompt_context.context.send_activity(reply) - # return False - # - # dialogs.add(TextPrompt("text_prompt", prompt_validator)) - # - # async def step_1(step_context: WaterfallStepContext) -> DialogTurnStatus: - # assert isinstance(step_context.active_dialog.state["stepIndex"], int) - # await step_context.context.send_activity("step1") - # return Dialog.end_of_turn - # - # async def step_2(step_context: WaterfallStepContext) -> None: - # assert isinstance(step_context.active_dialog.state["stepIndex"], int) - # await step_context.prompt("text_prompt", {"prompt": MessageFactory.text("Please type your name.")}) - # - # async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: - # assert isinstance(step_context.active_dialog.state["stepIndex"], int) - # await step_context.context.send_activity("step3") - # return Dialog.end_of_turn - # - # steps = [step_1, step_2, step_3] - # - # dialogs.add(WaterfallDialog("waterfall_dialog", steps)) - # - # step1 = await adapter.send("hello") - # step2 = await step1.assert_reply("step1") - # step3 = await step2.send("hello") - # step4 = await step3.assert_reply("Please type your name") - # step5 = await step4.send("hi") - # step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 1") - # step7 = await step6.send("hi") - # step8 = await step7.assert_reply("Please send a name that is longer than 3 characters. 2") - # step9 = await step8.send("hi") - # step10 = await step9.assert_reply("Please send a name that is longer than 3 characters. 3") - # step11 = await step10.send("Kyle") - # step12 = await step11.assert_reply("You got it at the 4th try!") - # await step12.assert_reply("step3") - # - # return True + @staticmethod + async def proceeds_through_waterfall(storage) -> bool: + # storage = MemoryStorage() + # BLOB_STORAGE_SETTINGS = BlobStorageSettings( + # account_key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", container_name="test", connection_string="AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + # ) + # storage = BlobStorage(BLOB_STORAGE_SETTINGS) + convo_state = ConversationState(storage) + + dialog_state = convo_state.create_property("dialogState") + dialogs = DialogSet(dialog_state) + + async def exec_test(turn_context: TurnContext) -> None: + dc = await dialogs.create_context(turn_context) + + await dc.continue_dialog() + if not turn_context.responded: + await dc.begin_dialog(WaterfallDialog.__name__) + await convo_state.save_changes(turn_context) + + adapter = TestAdapter(exec_test) + + async def prompt_validator(prompt_context: PromptValidatorContext): + result = prompt_context.recognized.value + if len(result) > 3: + succeeded_message = MessageFactory.text(f"You got it at the {prompt_context.options.number_of_attempts}rd try!") + await prompt_context.context.send_activity(succeeded_message) + return True + + reply = MessageFactory.text( + f"Please send a name that is longer than 3 characters. {prompt_context.options.number_of_attempts}" + ) + await prompt_context.context.send_activity(reply) + return False + + async def step_1(step_context: WaterfallStepContext) -> DialogTurnStatus: + assert isinstance(step_context.active_dialog.state["stepIndex"], int) + await step_context.context.send_activity("step1") + return Dialog.end_of_turn + + async def step_2(step_context: WaterfallStepContext) -> None: + assert isinstance(step_context.active_dialog.state["stepIndex"], int) + await step_context.prompt(TextPrompt.__name__, PromptOptions( + prompt=MessageFactory.text("Please type your name") + )) + + async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: + assert isinstance(step_context.active_dialog.state["stepIndex"], int) + await step_context.context.send_activity("step3") + return Dialog.end_of_turn + + steps = [step_1, step_2, step_3] + + dialogs.add(WaterfallDialog(WaterfallDialog.__name__, steps)) + + dialogs.add(TextPrompt(TextPrompt.__name__, prompt_validator)) + + step1 = await adapter.send("hello") # None + step2 = await step1.assert_reply("step1") # None + step3 = await step2.send("hello") # Tab1 + step4 = await step3.assert_reply("Please type your name") # None + step5 = await step4.send("hi") # Tab2 + step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 0") # None + step7 = await step6.send("hi") + step8 = await step7.assert_reply("Please send a name that is longer than 3 characters. 1") + step9 = await step8.send("hi") + step10 = await step9.assert_reply("Please send a name that is longer than 3 characters. 2") + step11 = await step10.send("Kyle") + step12 = await step11.assert_reply("You got it at the 3rd try!") + await step12.assert_reply("step3") + + return True From 97ea96ef590304e4ab93c7e609eeea7093f9cd7c Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Tue, 26 Nov 2019 11:56:06 -0800 Subject: [PATCH 04/11] removed commented code --- .../tests/storage_base_tests.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-core/tests/storage_base_tests.py index ea1c60776..84d9f2462 100644 --- a/libraries/botbuilder-core/tests/storage_base_tests.py +++ b/libraries/botbuilder-core/tests/storage_base_tests.py @@ -9,8 +9,7 @@ TODO: write example """ import pytest -from botbuilder.azure import BlobStorage, BlobStorageSettings -from botbuilder.core import ConversationState, TurnContext, MessageFactory, MemoryStorage +from botbuilder.core import ConversationState, TurnContext, MessageFactory from botbuilder.core.adapters import TestAdapter from botbuilder.dialogs import DialogSet, DialogTurnStatus, TextPrompt, PromptValidatorContext, WaterfallStepContext, \ Dialog, WaterfallDialog, PromptOptions @@ -237,11 +236,6 @@ async def perform_batch_operations(storage) -> bool: @staticmethod async def proceeds_through_waterfall(storage) -> bool: - # storage = MemoryStorage() - # BLOB_STORAGE_SETTINGS = BlobStorageSettings( - # account_key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", container_name="test", connection_string="AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" - # ) - # storage = BlobStorage(BLOB_STORAGE_SETTINGS) convo_state = ConversationState(storage) dialog_state = convo_state.create_property("dialogState") @@ -292,12 +286,12 @@ async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: dialogs.add(TextPrompt(TextPrompt.__name__, prompt_validator)) - step1 = await adapter.send("hello") # None - step2 = await step1.assert_reply("step1") # None - step3 = await step2.send("hello") # Tab1 + step1 = await adapter.send("hello") + step2 = await step1.assert_reply("step1") + step3 = await step2.send("hello") step4 = await step3.assert_reply("Please type your name") # None - step5 = await step4.send("hi") # Tab2 - step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 0") # None + step5 = await step4.send("hi") + step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 0") step7 = await step6.send("hi") step8 = await step7.assert_reply("Please send a name that is longer than 3 characters. 1") step9 = await step8.send("hi") From 348e3c3b5dbc39077a398e4735cee3f2b65a125d Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Tue, 26 Nov 2019 15:14:55 -0800 Subject: [PATCH 05/11] cosmos tests pass with storage_base_tests --- .../botbuilder/azure/cosmosdb_storage.py | 15 ++- .../tests/test_cosmos_partitioned_storage.py | 45 +++---- .../tests/test_cosmos_storage.py | 115 +++--------------- .../tests/storage_base_tests.py | 21 +++- 4 files changed, 60 insertions(+), 136 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py index d6f43c850..ec0200433 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py @@ -14,7 +14,7 @@ from jsonpickle.unpickler import Unpickler import azure.cosmos.cosmos_client as cosmos_client # pylint: disable=no-name-in-module,import-error import azure.cosmos.errors as cosmos_errors # pylint: disable=no-name-in-module,import-error -from botbuilder.core.storage import Storage +from botbuilder.core.storage import Storage, StoreItem class CosmosDbConfig: @@ -160,6 +160,10 @@ async def write(self, changes: Dict[str, object]): :param changes: :return: """ + if changes is None: + raise Exception("Changes are required when writing") + elif not changes: + return try: # check if the database and container exists and if not create if not self.__container_exists: @@ -167,15 +171,17 @@ async def write(self, changes: Dict[str, object]): # iterate over the changes for (key, change) in changes.items(): # store the e_tag - e_tag = change.get("e_tag", None) + e_tag = change.e_tag if isinstance(change, StoreItem) else change.get("e_tag", None) # create the new document doc = { "id": CosmosDbKeyEscape.sanitize_key(key), "realId": key, "document": self.__create_dict(change), } + if e_tag == "": + raise Exception("cosmosdb_storage.write(): etag missing") # the e_tag will be * for new docs so do an insert - if e_tag == "*" or not e_tag: + elif e_tag == "*" or not e_tag: self.client.UpsertItem( database_or_Container_link=self.__container_link, document=doc, @@ -191,9 +197,6 @@ async def write(self, changes: Dict[str, object]): new_document=doc, options={"accessCondition": access_condition}, ) - # error when there is no e_tag - else: - raise Exception("cosmosdb_storage.write(): etag missing") except Exception as error: raise error diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index 7b4e017ff..62f263c44 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -1,16 +1,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from unittest.mock import Mock import azure.cosmos.errors as cosmos_errors from azure.cosmos import documents -from azure.cosmos.cosmos_client import CosmosClient import pytest -from botbuilder.core import StoreItem from botbuilder.azure import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig from tests import StorageBaseTests -EMULATOR_RUNNING = True +EMULATOR_RUNNING = False def get_settings() -> CosmosDbPartitionedConfig: @@ -35,31 +32,8 @@ async def reset(): pass -def get_mock_client(identifier: str = "1"): - # pylint: disable=attribute-defined-outside-init, invalid-name - mock = MockClient() - - mock.QueryDatabases = Mock(return_value=[]) - mock.QueryContainers = Mock(return_value=[]) - mock.CreateDatabase = Mock(return_value={"id": identifier}) - mock.CreateContainer = Mock(return_value={"id": identifier}) - - return mock - - -class MockClient(CosmosClient): - def __init__(self): # pylint: disable=super-init-not-called - pass - - -class SimpleStoreItem(StoreItem): - def __init__(self, counter=1, e_tag="*"): - super(SimpleStoreItem, self).__init__() - self.counter = counter - self.e_tag = e_tag - - class TestCosmosDbPartitionedStorageConstructor: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_raises_error_when_instantiated_with_no_arguments(self): try: @@ -68,6 +42,7 @@ async def test_raises_error_when_instantiated_with_no_arguments(self): except Exception as e: assert e + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_raises_error_when_no_endpoint_provided(self): no_endpoint = get_settings() @@ -77,6 +52,7 @@ async def test_raises_error_when_no_endpoint_provided(self): except Exception as e: assert e + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_raises_error_when_no_auth_key_provided(self): no_auth_key = get_settings() @@ -86,6 +62,7 @@ async def test_raises_error_when_no_auth_key_provided(self): except Exception as e: assert e + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_raises_error_when_no_database_id_provided(self): no_database_id = get_settings() @@ -95,6 +72,7 @@ async def test_raises_error_when_no_database_id_provided(self): except Exception as e: assert e + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_raises_error_when_no_container_id_provided(self): no_container_id = get_settings() @@ -104,6 +82,7 @@ async def test_raises_error_when_no_container_id_provided(self): except Exception as e: assert e + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_passes_cosmos_client_options(self): settings_with_options = get_settings() @@ -124,6 +103,7 @@ async def test_passes_cosmos_client_options(self): class TestCosmosDbPartitionedStorageBaseStorageTests: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_return_empty_object_when_reading_unknown_key(self): await reset() @@ -132,6 +112,7 @@ async def test_return_empty_object_when_reading_unknown_key(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_reading(self): await reset() @@ -140,6 +121,7 @@ async def test_handle_null_keys_when_reading(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_writing(self): await reset() @@ -148,6 +130,7 @@ async def test_handle_null_keys_when_writing(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_does_not_raise_when_writing_no_items(self): await reset() @@ -156,6 +139,7 @@ async def test_does_not_raise_when_writing_no_items(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_create_object(self): await reset() @@ -164,6 +148,7 @@ async def test_create_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_crazy_keys(self): await reset() @@ -172,6 +157,7 @@ async def test_handle_crazy_keys(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_update_object(self): await reset() @@ -180,6 +166,7 @@ async def test_update_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_delete_object(self): await reset() @@ -188,6 +175,7 @@ async def test_delete_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_perform_batch_operations(self): await reset() @@ -196,6 +184,7 @@ async def test_perform_batch_operations(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_proceeds_through_waterfall(self): await reset() diff --git a/libraries/botbuilder-azure/tests/test_cosmos_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_storage.py index 3f1a7d125..f32da7560 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_storage.py @@ -7,10 +7,9 @@ import pytest from botbuilder.core import StoreItem from botbuilder.azure import CosmosDbStorage, CosmosDbConfig - -# local cosmosdb emulator instance cosmos_db_config from tests import StorageBaseTests +# local cosmosdb emulator instance cosmos_db_config COSMOS_DB_CONFIG = CosmosDbConfig( endpoint="https://localhost:8081", masterkey="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", @@ -65,7 +64,7 @@ async def test_cosmos_storage_init_should_error_without_cosmos_db_config(self): assert error @pytest.mark.asyncio - async def test_creation_request_options_era_being_called(self): + async def test_creation_request_options_are_being_called(self): # pylint: disable=protected-access test_config = CosmosDbConfig( endpoint="https://localhost:8081", @@ -94,14 +93,18 @@ async def test_creation_request_options_era_being_called(self): class TestCosmosDbStorageBaseStorageTests: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_return_empty_object_when_reading_unknown_key(self): await reset() - test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key(get_storage()) + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + get_storage() + ) assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_reading(self): await reset() @@ -110,6 +113,7 @@ async def test_handle_null_keys_when_reading(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_writing(self): await reset() @@ -118,14 +122,18 @@ async def test_handle_null_keys_when_writing(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_does_not_raise_when_writing_no_items(self): await reset() - test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items(get_storage()) + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + get_storage() + ) assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_create_object(self): await reset() @@ -134,6 +142,7 @@ async def test_create_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_crazy_keys(self): await reset() @@ -142,6 +151,7 @@ async def test_handle_crazy_keys(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_update_object(self): await reset() @@ -150,6 +160,7 @@ async def test_update_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_delete_object(self): await reset() @@ -158,6 +169,7 @@ async def test_delete_object(self): assert test_ran + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_perform_batch_operations(self): await reset() @@ -166,7 +178,7 @@ async def test_perform_batch_operations(self): assert test_ran - # TODO: Re-enable after the dialog_stack PR gets merged + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_proceeds_through_waterfall(self): await reset() @@ -177,8 +189,6 @@ async def test_proceeds_through_waterfall(self): class TestCosmosDbStorage: - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_cosmos_storage_init_should_work_with_just_endpoint_and_key(self): @@ -193,18 +203,6 @@ async def test_cosmos_storage_init_should_work_with_just_endpoint_and_key(self): assert data["user"].counter == 1 assert len(data.keys()) == 1 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_read_should_return_data_with_valid_key(self): - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - await storage.write({"user": SimpleStoreItem()}) - - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 - assert len(data.keys()) == 1 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_cosmos_storage_read_update_should_return_new_etag(self): @@ -228,27 +226,6 @@ async def test_cosmos_storage_read_with_invalid_key_should_return_empty_dict(sel assert isinstance(data, dict) assert not data.keys() - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_read_no_key_should_throw(self): - try: - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - await storage.read([]) - except Exception as error: - assert error - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_write_should_add_new_value(self): - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - await storage.write({"user": SimpleStoreItem(counter=1)}) - - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_cosmos_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk( @@ -262,62 +239,6 @@ async def test_cosmos_storage_write_should_overwrite_when_new_e_tag_is_an_asteri data = await storage.read(["user"]) assert data["user"].counter == 10 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_write_batch_operation(self): - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - await storage.write( - { - "batch1": SimpleStoreItem(counter=1), - "batch2": SimpleStoreItem(counter=1), - "batch3": SimpleStoreItem(counter=1), - } - ) - data = await storage.read(["batch1", "batch2", "batch3"]) - assert len(data.keys()) == 3 - assert data["batch1"] - assert data["batch2"] - assert data["batch3"] - assert data["batch1"].counter == 1 - assert data["batch2"].counter == 1 - assert data["batch3"].counter == 1 - assert data["batch1"].e_tag - assert data["batch2"].e_tag - assert data["batch3"].e_tag - await storage.delete(["batch1", "batch2", "batch3"]) - data = await storage.read(["batch1", "batch2", "batch3"]) - assert not data.keys() - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_write_crazy_keys_work(self): - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - crazy_key = '!@#$%^&*()_+??><":QASD~`' - await storage.write({crazy_key: SimpleStoreItem(counter=1)}) - data = await storage.read([crazy_key]) - assert len(data.keys()) == 1 - assert data[crazy_key] - assert data[crazy_key].counter == 1 - assert data[crazy_key].e_tag - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_cosmos_storage_delete_should_delete_according_cached_data(self): - await reset() - storage = CosmosDbStorage(COSMOS_DB_CONFIG) - await storage.write({"test": SimpleStoreItem()}) - try: - await storage.delete(["test"]) - except Exception as error: - raise error - else: - data = await storage.read(["test"]) - - assert isinstance(data, dict) - assert not data.keys() - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_cosmos_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-core/tests/storage_base_tests.py index 84d9f2462..d44a11644 100644 --- a/libraries/botbuilder-core/tests/storage_base_tests.py +++ b/libraries/botbuilder-core/tests/storage_base_tests.py @@ -6,10 +6,16 @@ Therefore, all tests using theses static tests should strictly check that the method returns true. :Example: -TODO: write example + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran """ import pytest -from botbuilder.core import ConversationState, TurnContext, MessageFactory +from botbuilder.azure import CosmosDbStorage +from botbuilder.core import ConversationState, TurnContext, MessageFactory, MemoryStorage from botbuilder.core.adapters import TestAdapter from botbuilder.dialogs import DialogSet, DialogTurnStatus, TextPrompt, PromptValidatorContext, WaterfallStepContext, \ Dialog, WaterfallDialog, PromptOptions @@ -27,9 +33,14 @@ async def return_empty_object_when_reading_unknown_key(storage) -> bool: @staticmethod async def handle_null_keys_when_reading(storage) -> bool: - with pytest.raises(Exception) as err: - await storage.read(None) - assert err.value.args[0] == "Keys are required when reading" + if isinstance(storage, CosmosDbStorage): + result = await storage.read(None) + assert len(result.keys()) == 0 + # Catch-all + else: + with pytest.raises(Exception) as err: + await storage.read(None) + assert err.value.args[0] == "Keys are required when reading" return True From f5837d5836d5392af136ea107406ce578ff4a9f9 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 10:55:50 -0800 Subject: [PATCH 06/11] blob storage uses storage_base_tests --- .../botbuilder/azure/blob_storage.py | 34 ++-- .../botbuilder/azure/cosmosdb_storage.py | 4 +- .../tests/test_blob_storage.py | 165 ++++++++++++------ 3 files changed, 132 insertions(+), 71 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py index ae3ad1766..d77235b80 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py @@ -42,7 +42,7 @@ def __init__(self, settings: BlobStorageSettings): async def read(self, keys: List[str]) -> Dict[str, object]: if not keys: - raise Exception("Please provide at least one key to read from storage.") + raise Exception("Keys are required when reading") self.client.create_container(self.settings.container_name) self.client.set_container_acl( @@ -63,24 +63,31 @@ async def read(self, keys: List[str]) -> Dict[str, object]: return items async def write(self, changes: Dict[str, object]): + if changes is None: + raise Exception("Changes are required when writing") + elif not changes: + return + self.client.create_container(self.settings.container_name) self.client.set_container_acl( self.settings.container_name, public_access=PublicAccess.Container ) - for name, item in changes.items(): - e_tag = ( - None if not hasattr(item, "e_tag") or item.e_tag == "*" else item.e_tag - ) - if e_tag: - item.e_tag = e_tag.replace('"', '\\"') + for (name, item) in changes.items(): + e_tag = item.e_tag if hasattr(item, "e_tag") else item.get("e_tag", None) + e_tag = None if e_tag == "*" else e_tag + if e_tag == "": + raise Exception("blob_storage.write(): etag missing") item_str = self._store_item_to_str(item) - self.client.create_blob_from_text( - container_name=self.settings.container_name, - blob_name=name, - text=item_str, - if_match=e_tag, - ) + try: + self.client.create_blob_from_text( + container_name=self.settings.container_name, + blob_name=name, + text=item_str, + if_match=e_tag, + ) + except Exception as error: + raise error async def delete(self, keys: List[str]): if keys is None: @@ -102,7 +109,6 @@ async def delete(self, keys: List[str]): def _blob_to_store_item(self, blob: Blob) -> object: item = json.loads(blob.content) item["e_tag"] = blob.properties.etag - item["id"] = blob.name result = Unpickler().restore(item) return result diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py index ec0200433..0bd68e993 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py @@ -14,7 +14,7 @@ from jsonpickle.unpickler import Unpickler import azure.cosmos.cosmos_client as cosmos_client # pylint: disable=no-name-in-module,import-error import azure.cosmos.errors as cosmos_errors # pylint: disable=no-name-in-module,import-error -from botbuilder.core.storage import Storage, StoreItem +from botbuilder.core.storage import Storage class CosmosDbConfig: @@ -171,7 +171,7 @@ async def write(self, changes: Dict[str, object]): # iterate over the changes for (key, change) in changes.items(): # store the e_tag - e_tag = change.e_tag if isinstance(change, StoreItem) else change.get("e_tag", None) + e_tag = change.e_tag if hasattr(change, "e_tag") else change.get("e_tag", None) # create the new document doc = { "id": CosmosDbKeyEscape.sanitize_key(key), diff --git a/libraries/botbuilder-azure/tests/test_blob_storage.py b/libraries/botbuilder-azure/tests/test_blob_storage.py index 40db0f61e..69d752f6e 100644 --- a/libraries/botbuilder-azure/tests/test_blob_storage.py +++ b/libraries/botbuilder-azure/tests/test_blob_storage.py @@ -6,14 +6,27 @@ from botbuilder.azure import BlobStorage, BlobStorageSettings # local blob emulator instance blob +from tests import StorageBaseTests + BLOB_STORAGE_SETTINGS = BlobStorageSettings( - account_name="", account_key="", container_name="test" + account_name="", + account_key="", + container_name="test", + # Default Azure Storage Emulator Connection String + connection_string="AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" ) EMULATOR_RUNNING = False +def get_storage(): + return BlobStorage(BLOB_STORAGE_SETTINGS) + + async def reset(): - storage = BlobStorage(BLOB_STORAGE_SETTINGS) + storage = get_storage() try: await storage.client.delete_container( container_name=BLOB_STORAGE_SETTINGS.container_name @@ -29,7 +42,7 @@ def __init__(self, counter=1, e_tag="*"): self.e_tag = e_tag -class TestBlobStorage: +class TestBlobStorageConstructor: @pytest.mark.asyncio async def test_blob_storage_init_should_error_without_cosmos_db_config(self): try: @@ -37,17 +50,104 @@ async def test_blob_storage_init_should_error_without_cosmos_db_config(self): except Exception as error: assert error + +class TestBlobStorageBaseTests: @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_blob_storage_read_should_return_data_with_valid_key(self): - storage = BlobStorage(BLOB_STORAGE_SETTINGS) - await storage.write({"user": SimpleStoreItem()}) + async def test_return_empty_object_when_reading_unknown_key(self): + await reset() - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 - assert len(data.keys()) == 1 + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + get_storage() + ) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_writing(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_does_not_raise_when_writing_no_items(self): + await reset() + + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + get_storage() + ) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_create_object(self): + await reset() + + test_ran = await StorageBaseTests.create_object(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_crazy_keys(self): + await reset() + + test_ran = await StorageBaseTests.handle_crazy_keys(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_update_object(self): + await reset() + test_ran = await StorageBaseTests.update_object(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_delete_object(self): + await reset() + + test_ran = await StorageBaseTests.delete_object(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_perform_batch_operations(self): + await reset() + + test_ran = await StorageBaseTests.perform_batch_operations(get_storage()) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_proceeds_through_waterfall(self): + await reset() + + test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) + + assert test_ran + + +class TestBlobStorage: @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_blob_storage_read_update_should_return_new_etag(self): @@ -60,25 +160,6 @@ async def test_blob_storage_read_update_should_return_new_etag(self): assert data_updated["test"].counter == 2 assert data_updated["test"].e_tag != data_result["test"].e_tag - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_read_no_key_should_throw(self): - try: - storage = BlobStorage(BLOB_STORAGE_SETTINGS) - await storage.read([]) - except Exception as error: - assert error - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_write_should_add_new_value(self): - storage = BlobStorage(BLOB_STORAGE_SETTINGS) - await storage.write({"user": SimpleStoreItem(counter=1)}) - - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk( @@ -91,32 +172,6 @@ async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk data = await storage.read(["user"]) assert data["user"].counter == 10 - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_write_batch_operation(self): - storage = BlobStorage(BLOB_STORAGE_SETTINGS) - await storage.write( - { - "batch1": SimpleStoreItem(counter=1), - "batch2": SimpleStoreItem(counter=1), - "batch3": SimpleStoreItem(counter=1), - } - ) - data = await storage.read(["batch1", "batch2", "batch3"]) - assert len(data.keys()) == 3 - assert data["batch1"] - assert data["batch2"] - assert data["batch3"] - assert data["batch1"].counter == 1 - assert data["batch2"].counter == 1 - assert data["batch3"].counter == 1 - assert data["batch1"].e_tag - assert data["batch2"].e_tag - assert data["batch3"].e_tag - await storage.delete(["batch1", "batch2", "batch3"]) - data = await storage.read(["batch1", "batch2", "batch3"]) - assert not data.keys() - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_blob_storage_delete_should_delete_according_cached_data(self): From 1c723e5d44058300e0f44568d40fc878b86bb102 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 12:12:05 -0800 Subject: [PATCH 07/11] memory_storage uses storage_base_tests --- .../botbuilder/core/memory_storage.py | 39 +++++--- .../tests/storage_base_tests.py | 2 +- .../tests/test_memory_storage.py | 98 +++++++++++++++---- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/libraries/botbuilder-core/botbuilder/core/memory_storage.py b/libraries/botbuilder-core/botbuilder/core/memory_storage.py index 73ff77bc4..d22f62042 100644 --- a/libraries/botbuilder-core/botbuilder/core/memory_storage.py +++ b/libraries/botbuilder-core/botbuilder/core/memory_storage.py @@ -22,6 +22,8 @@ async def delete(self, keys: List[str]): async def read(self, keys: List[str]): data = {} + if not keys: + return data try: for key in keys: if key in self.memory: @@ -32,10 +34,14 @@ async def read(self, keys: List[str]): return data async def write(self, changes: Dict[str, StoreItem]): + if changes is None: + raise Exception("Changes are required when writing") + elif not changes: + return try: # iterate over the changes for (key, change) in changes.items(): - new_value = change + new_value = deepcopy(change) old_state_etag = None # Check if the a matching key already exists in self.memory @@ -43,26 +49,31 @@ async def write(self, changes: Dict[str, StoreItem]): if key in self.memory: old_state = self.memory[key] if not isinstance(old_state, StoreItem): - if "eTag" in old_state: - old_state_etag = old_state["eTag"] + old_state_etag = old_state.get("e_tag", None) elif old_state.e_tag: old_state_etag = old_state.e_tag new_state = new_value # Set ETag if applicable - if hasattr(new_value, "e_tag"): - if ( - old_state_etag is not None - and new_value.e_tag != "*" - and new_value.e_tag < old_state_etag - ): - raise KeyError( - "Etag conflict.\nOriginal: %s\r\nCurrent: %s" - % (new_value.e_tag, old_state_etag) - ) + new_value_etag = new_value.e_tag if hasattr(new_value, "e_tag") else new_value.get("e_tag", None) + if new_value_etag == "": + raise Exception("blob_storage.write(): etag missing") + if ( + old_state_etag is not None + and new_value_etag is not None + and new_value_etag != "*" + and new_value_etag < old_state_etag + ): + raise KeyError( + "Etag conflict.\nOriginal: %s\r\nCurrent: %s" + % (new_value_etag, old_state_etag) + ) + if hasattr(new_state, "e_tag"): new_state.e_tag = str(self._e_tag) - self._e_tag += 1 + else: + new_state["e_tag"] = str(self._e_tag) + self._e_tag += 1 self.memory[key] = deepcopy(new_state) except Exception as error: diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-core/tests/storage_base_tests.py index d44a11644..7c192ec64 100644 --- a/libraries/botbuilder-core/tests/storage_base_tests.py +++ b/libraries/botbuilder-core/tests/storage_base_tests.py @@ -33,7 +33,7 @@ async def return_empty_object_when_reading_unknown_key(storage) -> bool: @staticmethod async def handle_null_keys_when_reading(storage) -> bool: - if isinstance(storage, CosmosDbStorage): + if isinstance(storage, CosmosDbStorage) or isinstance(storage, MemoryStorage): result = await storage.read(None) assert len(result.keys()) == 0 # Catch-all diff --git a/libraries/botbuilder-core/tests/test_memory_storage.py b/libraries/botbuilder-core/tests/test_memory_storage.py index 63946ad60..37cd07e4d 100644 --- a/libraries/botbuilder-core/tests/test_memory_storage.py +++ b/libraries/botbuilder-core/tests/test_memory_storage.py @@ -1,9 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import aiounittest +import pytest from botbuilder.core import MemoryStorage, StoreItem +from tests import StorageBaseTests + + +def get_storage(): + return MemoryStorage() class SimpleStoreItem(StoreItem): @@ -13,7 +18,7 @@ def __init__(self, counter=1, e_tag="*"): self.e_tag = e_tag -class TestMemoryStorage(aiounittest.AsyncTestCase): +class TestMemoryStorageConstructor: def test_initializing_memory_storage_without_data_should_still_have_memory(self): storage = MemoryStorage() assert storage.memory is not None @@ -23,6 +28,7 @@ def test_memory_storage__e_tag_should_start_at_0(self): storage = MemoryStorage() assert storage._e_tag == 0 # pylint: disable=protected-access + @pytest.mark.asyncio async def test_memory_storage_initialized_with_memory_should_have_accessible_data( self, ): @@ -32,26 +38,75 @@ async def test_memory_storage_initialized_with_memory_should_have_accessible_dat assert data["test"].counter == 1 assert len(data.keys()) == 1 - async def test_memory_storage_read_should_return_data_with_valid_key(self): - storage = MemoryStorage() - await storage.write({"user": SimpleStoreItem()}) - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 - assert len(data.keys()) == 1 - assert storage._e_tag == 1 # pylint: disable=protected-access - assert int(data["user"].e_tag) == 0 +class TestMemoryStorageBaseTests: + @pytest.mark.asyncio + async def test_return_empty_object_when_reading_unknown_key(self): + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + get_storage() + ) - async def test_memory_storage_write_should_add_new_value(self): - storage = MemoryStorage() - aux = {"user": SimpleStoreItem(counter=1)} - await storage.write(aux) + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self): + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self): + test_ran = await StorageBaseTests.handle_null_keys_when_writing(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_does_not_raise_when_writing_no_items(self): + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + get_storage() + ) + + assert test_ran + + @pytest.mark.asyncio + async def test_create_object(self): + test_ran = await StorageBaseTests.create_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_handle_crazy_keys(self): + test_ran = await StorageBaseTests.handle_crazy_keys(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_update_object(self): + test_ran = await StorageBaseTests.update_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_delete_object(self): + test_ran = await StorageBaseTests.delete_object(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_perform_batch_operations(self): + test_ran = await StorageBaseTests.perform_batch_operations(get_storage()) + + assert test_ran + + @pytest.mark.asyncio + async def test_proceeds_through_waterfall(self): + test_ran = await StorageBaseTests.proceeds_through_waterfall(get_storage()) + + assert test_ran - data = await storage.read(["user"]) - assert "user" in data - assert data["user"].counter == 1 +class TestMemoryStorage: + @pytest.mark.asyncio async def test_memory_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk_1( self, ): @@ -62,6 +117,7 @@ async def test_memory_storage_write_should_overwrite_when_new_e_tag_is_an_asteri data = await storage.read(["user"]) assert data["user"].counter == 10 + @pytest.mark.asyncio async def test_memory_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk_2( self, ): @@ -72,6 +128,7 @@ async def test_memory_storage_write_should_overwrite_when_new_e_tag_is_an_asteri data = await storage.read(["user"]) assert data["user"].counter == 5 + @pytest.mark.asyncio async def test_memory_storage_read_with_invalid_key_should_return_empty_dict(self): storage = MemoryStorage() data = await storage.read(["test"]) @@ -79,6 +136,7 @@ async def test_memory_storage_read_with_invalid_key_should_return_empty_dict(sel assert isinstance(data, dict) assert not data.keys() + @pytest.mark.asyncio async def test_memory_storage_delete_should_delete_according_cached_data(self): storage = MemoryStorage({"test": "test"}) try: @@ -91,6 +149,7 @@ async def test_memory_storage_delete_should_delete_according_cached_data(self): assert isinstance(data, dict) assert not data.keys() + @pytest.mark.asyncio async def test_memory_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( self, ): @@ -102,6 +161,7 @@ async def test_memory_storage_delete_should_delete_multiple_values_when_given_mu data = await storage.read(["test", "test2"]) assert not data.keys() + @pytest.mark.asyncio async def test_memory_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data( self, ): @@ -117,6 +177,7 @@ async def test_memory_storage_delete_should_delete_values_when_given_multiple_va data = await storage.read(["test", "test2", "test3"]) assert len(data.keys()) == 1 + @pytest.mark.asyncio async def test_memory_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data( self, ): @@ -128,6 +189,7 @@ async def test_memory_storage_delete_invalid_key_should_do_nothing_and_not_affec data = await storage.read(["foo"]) assert not data.keys() + @pytest.mark.asyncio async def test_memory_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data( self, ): From cc539b217d4ca8923b06552dfd4180c905de3626 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 13:46:50 -0800 Subject: [PATCH 08/11] attempt to fix storage_base_tests import --- .../botbuilder/azure/blob_storage.py | 1 - .../botbuilder-azure/tests/test_blob_storage.py | 2 +- .../tests/test_cosmos_partitioned_storage.py | 2 +- .../botbuilder-azure/tests/test_cosmos_storage.py | 2 +- .../botbuilder/core/memory_storage.py | 6 +++++- libraries/botbuilder-core/tests/__init__.py | 14 ++------------ .../botbuilder-core/tests/test_memory_storage.py | 2 +- 7 files changed, 11 insertions(+), 18 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py index d77235b80..3a5e9a93b 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py @@ -3,7 +3,6 @@ from jsonpickle import encode from jsonpickle.unpickler import Unpickler - from azure.storage.blob import BlockBlobService, Blob, PublicAccess from botbuilder.core import Storage diff --git a/libraries/botbuilder-azure/tests/test_blob_storage.py b/libraries/botbuilder-azure/tests/test_blob_storage.py index 69d752f6e..952a0605a 100644 --- a/libraries/botbuilder-azure/tests/test_blob_storage.py +++ b/libraries/botbuilder-azure/tests/test_blob_storage.py @@ -4,9 +4,9 @@ import pytest from botbuilder.core import StoreItem from botbuilder.azure import BlobStorage, BlobStorageSettings +from tests.storage_base_tests import StorageBaseTests # local blob emulator instance blob -from tests import StorageBaseTests BLOB_STORAGE_SETTINGS = BlobStorageSettings( account_name="", diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index 62f263c44..1a5e7dee2 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -5,7 +5,7 @@ from azure.cosmos import documents import pytest from botbuilder.azure import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig -from tests import StorageBaseTests +from tests.storage_base_tests import StorageBaseTests EMULATOR_RUNNING = False diff --git a/libraries/botbuilder-azure/tests/test_cosmos_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_storage.py index f32da7560..05ad3afc5 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_storage.py @@ -7,7 +7,7 @@ import pytest from botbuilder.core import StoreItem from botbuilder.azure import CosmosDbStorage, CosmosDbConfig -from tests import StorageBaseTests +from tests.storage_base_tests import StorageBaseTests # local cosmosdb emulator instance cosmos_db_config COSMOS_DB_CONFIG = CosmosDbConfig( diff --git a/libraries/botbuilder-core/botbuilder/core/memory_storage.py b/libraries/botbuilder-core/botbuilder/core/memory_storage.py index d22f62042..100d54ba7 100644 --- a/libraries/botbuilder-core/botbuilder/core/memory_storage.py +++ b/libraries/botbuilder-core/botbuilder/core/memory_storage.py @@ -56,7 +56,11 @@ async def write(self, changes: Dict[str, StoreItem]): new_state = new_value # Set ETag if applicable - new_value_etag = new_value.e_tag if hasattr(new_value, "e_tag") else new_value.get("e_tag", None) + new_value_etag = ( + new_value.e_tag + if hasattr(new_value, "e_tag") + else new_value.get("e_tag", None) + ) if new_value_etag == "": raise Exception("blob_storage.write(): etag missing") if ( diff --git a/libraries/botbuilder-core/tests/__init__.py b/libraries/botbuilder-core/tests/__init__.py index 2bef95eb6..2f5c78e48 100644 --- a/libraries/botbuilder-core/tests/__init__.py +++ b/libraries/botbuilder-core/tests/__init__.py @@ -1,12 +1,2 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from .storage_base_tests import StorageBaseTests - -__all__ = [ - "StorageBaseTests" -] +import pkgutil +__path__ = pkgutil.extend_path(__path__, __name__) \ No newline at end of file diff --git a/libraries/botbuilder-core/tests/test_memory_storage.py b/libraries/botbuilder-core/tests/test_memory_storage.py index 37cd07e4d..0f7217104 100644 --- a/libraries/botbuilder-core/tests/test_memory_storage.py +++ b/libraries/botbuilder-core/tests/test_memory_storage.py @@ -4,7 +4,7 @@ import pytest from botbuilder.core import MemoryStorage, StoreItem -from tests import StorageBaseTests +from tests.storage_base_tests import StorageBaseTests def get_storage(): From 9bf3f56fd9c440c0c496ff2c197f21a3cf334870 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 13:56:44 -0800 Subject: [PATCH 09/11] moved storage_base_tests --- libraries/botbuilder-azure/tests/test_blob_storage.py | 2 +- .../botbuilder-azure/tests/test_cosmos_partitioned_storage.py | 2 +- libraries/botbuilder-azure/tests/test_cosmos_storage.py | 2 +- libraries/botbuilder-core/tests/__init__.py | 2 -- libraries/botbuilder-core/tests/test_memory_storage.py | 2 +- libraries/botbuilder-testing/botbuilder/testing/__init__.py | 3 ++- .../botbuilder/testing}/storage_base_tests.py | 0 7 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 libraries/botbuilder-core/tests/__init__.py rename libraries/{botbuilder-core/tests => botbuilder-testing/botbuilder/testing}/storage_base_tests.py (100%) diff --git a/libraries/botbuilder-azure/tests/test_blob_storage.py b/libraries/botbuilder-azure/tests/test_blob_storage.py index 952a0605a..9cfdb78fc 100644 --- a/libraries/botbuilder-azure/tests/test_blob_storage.py +++ b/libraries/botbuilder-azure/tests/test_blob_storage.py @@ -4,7 +4,7 @@ import pytest from botbuilder.core import StoreItem from botbuilder.azure import BlobStorage, BlobStorageSettings -from tests.storage_base_tests import StorageBaseTests +from botbuilder.testing import StorageBaseTests # local blob emulator instance blob diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index 1a5e7dee2..c8c0abf1c 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -5,7 +5,7 @@ from azure.cosmos import documents import pytest from botbuilder.azure import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig -from tests.storage_base_tests import StorageBaseTests +from botbuilder.testing import StorageBaseTests EMULATOR_RUNNING = False diff --git a/libraries/botbuilder-azure/tests/test_cosmos_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_storage.py index 05ad3afc5..c66660857 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_storage.py @@ -7,7 +7,7 @@ import pytest from botbuilder.core import StoreItem from botbuilder.azure import CosmosDbStorage, CosmosDbConfig -from tests.storage_base_tests import StorageBaseTests +from botbuilder.testing import StorageBaseTests # local cosmosdb emulator instance cosmos_db_config COSMOS_DB_CONFIG = CosmosDbConfig( diff --git a/libraries/botbuilder-core/tests/__init__.py b/libraries/botbuilder-core/tests/__init__.py deleted file mode 100644 index 2f5c78e48..000000000 --- a/libraries/botbuilder-core/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import pkgutil -__path__ = pkgutil.extend_path(__path__, __name__) \ No newline at end of file diff --git a/libraries/botbuilder-core/tests/test_memory_storage.py b/libraries/botbuilder-core/tests/test_memory_storage.py index 0f7217104..a34e2a94e 100644 --- a/libraries/botbuilder-core/tests/test_memory_storage.py +++ b/libraries/botbuilder-core/tests/test_memory_storage.py @@ -4,7 +4,7 @@ import pytest from botbuilder.core import MemoryStorage, StoreItem -from tests.storage_base_tests import StorageBaseTests +from botbuilder.testing import StorageBaseTests def get_storage(): diff --git a/libraries/botbuilder-testing/botbuilder/testing/__init__.py b/libraries/botbuilder-testing/botbuilder/testing/__init__.py index 681a168e4..af82e1a65 100644 --- a/libraries/botbuilder-testing/botbuilder/testing/__init__.py +++ b/libraries/botbuilder-testing/botbuilder/testing/__init__.py @@ -3,6 +3,7 @@ from .dialog_test_client import DialogTestClient from .dialog_test_logger import DialogTestLogger +from .storage_base_tests import StorageBaseTests -__all__ = ["DialogTestClient", "DialogTestLogger"] +__all__ = ["DialogTestClient", "DialogTestLogger", "StorageBaseTests"] diff --git a/libraries/botbuilder-core/tests/storage_base_tests.py b/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py similarity index 100% rename from libraries/botbuilder-core/tests/storage_base_tests.py rename to libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py From d54f68969011282094da107e4026f90ae69ab305 Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 14:01:32 -0800 Subject: [PATCH 10/11] black compliance --- .../botbuilder/azure/__init__.py | 5 +- .../azure/cosmosdb_partitioned_storage.py | 28 ++++--- .../botbuilder/azure/cosmosdb_storage.py | 6 +- .../tests/test_blob_storage.py | 8 +- .../tests/test_cosmos_partitioned_storage.py | 17 +++-- .../botbuilder/testing/storage_base_tests.py | 73 ++++++++++++------- 6 files changed, 91 insertions(+), 46 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/__init__.py b/libraries/botbuilder-azure/botbuilder/azure/__init__.py index 703fc465c..9980f8aa4 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/__init__.py +++ b/libraries/botbuilder-azure/botbuilder/azure/__init__.py @@ -7,7 +7,10 @@ from .about import __version__ from .cosmosdb_storage import CosmosDbStorage, CosmosDbConfig, CosmosDbKeyEscape -from .cosmosdb_partitioned_storage import CosmosDbPartitionedStorage, CosmosDbPartitionedConfig +from .cosmosdb_partitioned_storage import ( + CosmosDbPartitionedStorage, + CosmosDbPartitionedConfig, +) from .blob_storage import BlobStorage, BlobStorageSettings __all__ = [ diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py index 8c7f046f4..6a9f0f5d8 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py @@ -62,10 +62,7 @@ def __init__( class CosmosDbPartitionedStorage(Storage): """The class for partitioned CosmosDB middleware for the Azure Bot Framework.""" - def __init__( - self, - config: CosmosDbPartitionedConfig - ): + def __init__(self, config: CosmosDbPartitionedConfig): """Create the storage object. :param config: @@ -99,12 +96,17 @@ async def read(self, keys: List[str]) -> Dict[str, object]: ) document_store_item = read_item_response if document_store_item: - store_items[document_store_item["realId"]] = self.__create_si(document_store_item) + store_items[document_store_item["realId"]] = self.__create_si( + document_store_item + ) # When an item is not found a CosmosException is thrown, but we want to # return an empty collection so in this instance we catch and do not rethrow. # Throw for any other exception. except cosmos_errors.HTTPFailure as err: - if err.status_code == cosmos_errors.http_constants.StatusCodes.NOT_FOUND: + if ( + err.status_code + == cosmos_errors.http_constants.StatusCodes.NOT_FOUND + ): continue else: raise err @@ -135,8 +137,12 @@ async def write(self, changes: Dict[str, object]): if e_tag == "": raise Exception("cosmosdb_storage.write(): etag missing") - accessCondition = {"accessCondition": {"type": "IfMatch", "condition": e_tag}} - options = accessCondition if e_tag != "*" and e_tag and e_tag != "" else None + accessCondition = { + "accessCondition": {"type": "IfMatch", "condition": e_tag} + } + options = ( + accessCondition if e_tag != "*" and e_tag and e_tag != "" else None + ) try: self.client.UpsertItem( database_or_Container_link=self.__container_link, @@ -164,7 +170,10 @@ async def delete(self, keys: List[str]): options={"partitionKey": escaped_key}, ) except cosmos_errors.HTTPFailure as err: - if err.status_code == cosmos_errors.http_constants.StatusCodes.NOT_FOUND: + if ( + err.status_code + == cosmos_errors.http_constants.StatusCodes.NOT_FOUND + ): continue else: raise err @@ -253,7 +262,6 @@ def __create_dict(store_item: object) -> Dict: # loop through attributes and write and return a dict return json_dict - def __item_link(self, identifier) -> str: """Return the item link of a item in the container. diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py index 0bd68e993..9ded9248d 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py @@ -171,7 +171,11 @@ async def write(self, changes: Dict[str, object]): # iterate over the changes for (key, change) in changes.items(): # store the e_tag - e_tag = change.e_tag if hasattr(change, "e_tag") else change.get("e_tag", None) + e_tag = ( + change.e_tag + if hasattr(change, "e_tag") + else change.get("e_tag", None) + ) # create the new document doc = { "id": CosmosDbKeyEscape.sanitize_key(key), diff --git a/libraries/botbuilder-azure/tests/test_blob_storage.py b/libraries/botbuilder-azure/tests/test_blob_storage.py index 9cfdb78fc..31f54a231 100644 --- a/libraries/botbuilder-azure/tests/test_blob_storage.py +++ b/libraries/botbuilder-azure/tests/test_blob_storage.py @@ -13,10 +13,10 @@ account_key="", container_name="test", # Default Azure Storage Emulator Connection String - connection_string="AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + - "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + - "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + - "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + connection_string="AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;", ) EMULATOR_RUNNING = False diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index c8c0abf1c..4c22a9ce1 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -15,7 +15,7 @@ def get_settings() -> CosmosDbPartitionedConfig: cosmos_db_endpoint="https://localhost:8081", auth_key="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", database_id="test-db", - container_id="bot-storage" + container_id="bot-storage", ) @@ -92,14 +92,17 @@ async def test_passes_cosmos_client_options(self): settings_with_options.cosmos_client_options = { "connection_policy": connection_policy, - "consistency_level": documents.ConsistencyLevel.Eventual + "consistency_level": documents.ConsistencyLevel.Eventual, } client = CosmosDbPartitionedStorage(settings_with_options) await client.initialize() assert client.client.connection_policy.DisableSSLVerification is True - assert client.client.default_headers['x-ms-consistency-level'] == documents.ConsistencyLevel.Eventual + assert ( + client.client.default_headers["x-ms-consistency-level"] + == documents.ConsistencyLevel.Eventual + ) class TestCosmosDbPartitionedStorageBaseStorageTests: @@ -108,7 +111,9 @@ class TestCosmosDbPartitionedStorageBaseStorageTests: async def test_return_empty_object_when_reading_unknown_key(self): await reset() - test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key(get_storage()) + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + get_storage() + ) assert test_ran @@ -135,7 +140,9 @@ async def test_handle_null_keys_when_writing(self): async def test_does_not_raise_when_writing_no_items(self): await reset() - test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items(get_storage()) + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + get_storage() + ) assert test_ran diff --git a/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py b/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py index 7c192ec64..939ee5f83 100644 --- a/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py +++ b/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py @@ -15,16 +15,29 @@ async def test_handle_null_keys_when_reading(self): """ import pytest from botbuilder.azure import CosmosDbStorage -from botbuilder.core import ConversationState, TurnContext, MessageFactory, MemoryStorage +from botbuilder.core import ( + ConversationState, + TurnContext, + MessageFactory, + MemoryStorage, +) from botbuilder.core.adapters import TestAdapter -from botbuilder.dialogs import DialogSet, DialogTurnStatus, TextPrompt, PromptValidatorContext, WaterfallStepContext, \ - Dialog, WaterfallDialog, PromptOptions +from botbuilder.dialogs import ( + DialogSet, + DialogTurnStatus, + TextPrompt, + PromptValidatorContext, + WaterfallStepContext, + Dialog, + WaterfallDialog, + PromptOptions, +) class StorageBaseTests: @staticmethod async def return_empty_object_when_reading_unknown_key(storage) -> bool: - result = await storage.read(['unknown']) + result = await storage.read(["unknown"]) assert result is not None assert len(result) == 0 @@ -74,7 +87,10 @@ async def create_object(storage) -> bool: read_store_items = await storage.read(store_items.keys()) assert store_items["createPoco"]["id"] == read_store_items["createPoco"]["id"] - assert store_items["createPocoStoreItem"]["id"] == read_store_items["createPocoStoreItem"]["id"] + assert ( + store_items["createPocoStoreItem"]["id"] + == read_store_items["createPocoStoreItem"]["id"] + ) assert read_store_items["createPoco"]["e_tag"] is not None assert read_store_items["createPocoStoreItem"]["e_tag"] is not None @@ -82,7 +98,7 @@ async def create_object(storage) -> bool: @staticmethod async def handle_crazy_keys(storage) -> bool: - key = '!@#$%^&*()~/\\><,.?\';"\`~' + key = "!@#$%^&*()~/\\><,.?';\"\`~" store_item = {"id": 1} store_items = {key: store_item} @@ -99,7 +115,7 @@ async def handle_crazy_keys(storage) -> bool: async def update_object(storage) -> bool: original_store_items = { "pocoItem": {"id": 1, "count": 1}, - "pocoStoreItem": {"id": 1, "count": 1} + "pocoStoreItem": {"id": 1, "count": 1}, } # 1st write should work @@ -124,7 +140,9 @@ async def update_object(storage) -> bool: reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] assert reloaded_update_poco_item["e_tag"] is not None - assert update_poco_store_item["e_tag"] != reloaded_update_poco_store_item["e_tag"] + assert ( + update_poco_store_item["e_tag"] != reloaded_update_poco_store_item["e_tag"] + ) assert reloaded_update_poco_item["count"] == 2 assert reloaded_update_poco_store_item["count"] == 2 @@ -155,7 +173,7 @@ async def update_object(storage) -> bool: wildcard_etag_dict = { "pocoItem": reloaded_poco_item2, - "pocoStoreItem": reloaded_poco_store_item2 + "pocoStoreItem": reloaded_poco_store_item2, } await storage.write(wildcard_etag_dict) @@ -186,9 +204,7 @@ async def update_object(storage) -> bool: @staticmethod async def delete_object(storage) -> bool: - store_items = { - "delete1": {"id": 1, "count": 1} - } + store_items = {"delete1": {"id": 1, "count": 1}} await storage.write(store_items) @@ -217,11 +233,9 @@ async def delete_unknown_object(storage) -> bool: @staticmethod async def perform_batch_operations(storage) -> bool: - await storage.write({ - "batch1": {"count": 10}, - "batch2": {"count": 20}, - "batch3": {"count": 30}, - }) + await storage.write( + {"batch1": {"count": 10}, "batch2": {"count": 20}, "batch3": {"count": 30},} + ) result = await storage.read(["batch1", "batch2", "batch3"]) @@ -265,7 +279,9 @@ async def exec_test(turn_context: TurnContext) -> None: async def prompt_validator(prompt_context: PromptValidatorContext): result = prompt_context.recognized.value if len(result) > 3: - succeeded_message = MessageFactory.text(f"You got it at the {prompt_context.options.number_of_attempts}rd try!") + succeeded_message = MessageFactory.text( + f"You got it at the {prompt_context.options.number_of_attempts}rd try!" + ) await prompt_context.context.send_activity(succeeded_message) return True @@ -282,9 +298,10 @@ async def step_1(step_context: WaterfallStepContext) -> DialogTurnStatus: async def step_2(step_context: WaterfallStepContext) -> None: assert isinstance(step_context.active_dialog.state["stepIndex"], int) - await step_context.prompt(TextPrompt.__name__, PromptOptions( - prompt=MessageFactory.text("Please type your name") - )) + await step_context.prompt( + TextPrompt.__name__, + PromptOptions(prompt=MessageFactory.text("Please type your name")), + ) async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: assert isinstance(step_context.active_dialog.state["stepIndex"], int) @@ -300,13 +317,19 @@ async def step_3(step_context: WaterfallStepContext) -> DialogTurnStatus: step1 = await adapter.send("hello") step2 = await step1.assert_reply("step1") step3 = await step2.send("hello") - step4 = await step3.assert_reply("Please type your name") # None + step4 = await step3.assert_reply("Please type your name") # None step5 = await step4.send("hi") - step6 = await step5.assert_reply("Please send a name that is longer than 3 characters. 0") + step6 = await step5.assert_reply( + "Please send a name that is longer than 3 characters. 0" + ) step7 = await step6.send("hi") - step8 = await step7.assert_reply("Please send a name that is longer than 3 characters. 1") + step8 = await step7.assert_reply( + "Please send a name that is longer than 3 characters. 1" + ) step9 = await step8.send("hi") - step10 = await step9.assert_reply("Please send a name that is longer than 3 characters. 2") + step10 = await step9.assert_reply( + "Please send a name that is longer than 3 characters. 2" + ) step11 = await step10.send("Kyle") step12 = await step11.assert_reply("You got it at the 3rd try!") await step12.assert_reply("step3") From 613ac37ab05e2f91aed788fd817d8ecddf5c57dc Mon Sep 17 00:00:00 2001 From: mdrichardson Date: Wed, 27 Nov 2019 14:27:12 -0800 Subject: [PATCH 11/11] pylint compliance --- .../botbuilder/azure/blob_storage.py | 2 +- .../azure/cosmosdb_partitioned_storage.py | 13 +++++------- .../botbuilder/azure/cosmosdb_storage.py | 4 ++-- .../tests/test_cosmos_partitioned_storage.py | 21 ++++++++++--------- .../botbuilder/core/bot_state.py | 2 +- .../botbuilder/core/memory_storage.py | 2 +- .../botbuilder/testing/storage_base_tests.py | 10 ++++----- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py index 3a5e9a93b..fada3fe53 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py @@ -64,7 +64,7 @@ async def read(self, keys: List[str]) -> Dict[str, object]: async def write(self, changes: Dict[str, object]): if changes is None: raise Exception("Changes are required when writing") - elif not changes: + if not changes: return self.client.create_container(self.settings.container_name) diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py index 6a9f0f5d8..00c3bb137 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_partitioned_storage.py @@ -6,7 +6,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from hashlib import sha256 from typing import Dict, List from threading import Semaphore import json @@ -108,8 +107,7 @@ async def read(self, keys: List[str]) -> Dict[str, object]: == cosmos_errors.http_constants.StatusCodes.NOT_FOUND ): continue - else: - raise err + raise err except Exception as err: raise err return store_items @@ -122,7 +120,7 @@ async def write(self, changes: Dict[str, object]): """ if changes is None: raise Exception("Changes are required when writing") - elif not changes: + if not changes: return await self.initialize() @@ -137,11 +135,11 @@ async def write(self, changes: Dict[str, object]): if e_tag == "": raise Exception("cosmosdb_storage.write(): etag missing") - accessCondition = { + access_condition = { "accessCondition": {"type": "IfMatch", "condition": e_tag} } options = ( - accessCondition if e_tag != "*" and e_tag and e_tag != "" else None + access_condition if e_tag != "*" and e_tag and e_tag != "" else None ) try: self.client.UpsertItem( @@ -175,8 +173,7 @@ async def delete(self, keys: List[str]): == cosmos_errors.http_constants.StatusCodes.NOT_FOUND ): continue - else: - raise err + raise err except Exception as err: raise err diff --git a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py index 9ded9248d..3d588a864 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py +++ b/libraries/botbuilder-azure/botbuilder/azure/cosmosdb_storage.py @@ -162,7 +162,7 @@ async def write(self, changes: Dict[str, object]): """ if changes is None: raise Exception("Changes are required when writing") - elif not changes: + if not changes: return try: # check if the database and container exists and if not create @@ -185,7 +185,7 @@ async def write(self, changes: Dict[str, object]): if e_tag == "": raise Exception("cosmosdb_storage.write(): etag missing") # the e_tag will be * for new docs so do an insert - elif e_tag == "*" or not e_tag: + if e_tag == "*" or not e_tag: self.client.UpsertItem( database_or_Container_link=self.__container_link, document=doc, diff --git a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py index 4c22a9ce1..cb6dd0822 100644 --- a/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py +++ b/libraries/botbuilder-azure/tests/test_cosmos_partitioned_storage.py @@ -38,9 +38,10 @@ class TestCosmosDbPartitionedStorageConstructor: async def test_raises_error_when_instantiated_with_no_arguments(self): try: # noinspection PyArgumentList + # pylint: disable=no-value-for-parameter CosmosDbPartitionedStorage() - except Exception as e: - assert e + except Exception as error: + assert error @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio @@ -49,8 +50,8 @@ async def test_raises_error_when_no_endpoint_provided(self): no_endpoint.cosmos_db_endpoint = None try: CosmosDbPartitionedStorage(no_endpoint) - except Exception as e: - assert e + except Exception as error: + assert error @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio @@ -59,8 +60,8 @@ async def test_raises_error_when_no_auth_key_provided(self): no_auth_key.auth_key = None try: CosmosDbPartitionedStorage(no_auth_key) - except Exception as e: - assert e + except Exception as error: + assert error @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio @@ -69,8 +70,8 @@ async def test_raises_error_when_no_database_id_provided(self): no_database_id.database_id = None try: CosmosDbPartitionedStorage(no_database_id) - except Exception as e: - assert e + except Exception as error: + assert error @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio @@ -79,8 +80,8 @@ async def test_raises_error_when_no_container_id_provided(self): no_container_id.container_id = None try: CosmosDbPartitionedStorage(no_container_id) - except Exception as e: - assert e + except Exception as error: + assert error @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio diff --git a/libraries/botbuilder-core/botbuilder/core/bot_state.py b/libraries/botbuilder-core/botbuilder/core/bot_state.py index 4c9a178b4..4e615dda0 100644 --- a/libraries/botbuilder-core/botbuilder/core/bot_state.py +++ b/libraries/botbuilder-core/botbuilder/core/bot_state.py @@ -4,11 +4,11 @@ from abc import abstractmethod from copy import deepcopy from typing import Callable, Dict, Union +from jsonpickle.pickler import Pickler from botbuilder.core.state_property_accessor import StatePropertyAccessor from .turn_context import TurnContext from .storage import Storage from .property_manager import PropertyManager -from jsonpickle.pickler import Pickler class CachedBotState: diff --git a/libraries/botbuilder-core/botbuilder/core/memory_storage.py b/libraries/botbuilder-core/botbuilder/core/memory_storage.py index 100d54ba7..b85b3d368 100644 --- a/libraries/botbuilder-core/botbuilder/core/memory_storage.py +++ b/libraries/botbuilder-core/botbuilder/core/memory_storage.py @@ -36,7 +36,7 @@ async def read(self, keys: List[str]): async def write(self, changes: Dict[str, StoreItem]): if changes is None: raise Exception("Changes are required when writing") - elif not changes: + if not changes: return try: # iterate over the changes diff --git a/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py b/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py index 939ee5f83..defa5040f 100644 --- a/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py +++ b/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py @@ -46,7 +46,7 @@ async def return_empty_object_when_reading_unknown_key(storage) -> bool: @staticmethod async def handle_null_keys_when_reading(storage) -> bool: - if isinstance(storage, CosmosDbStorage) or isinstance(storage, MemoryStorage): + if isinstance(storage, (CosmosDbStorage, MemoryStorage)): result = await storage.read(None) assert len(result.keys()) == 0 # Catch-all @@ -98,7 +98,7 @@ async def create_object(storage) -> bool: @staticmethod async def handle_crazy_keys(storage) -> bool: - key = "!@#$%^&*()~/\\><,.?';\"\`~" + key = '!@#$%^&*()_+??><":QASD~`' store_item = {"id": 1} store_items = {key: store_item} @@ -267,11 +267,11 @@ async def proceeds_through_waterfall(storage) -> bool: dialogs = DialogSet(dialog_state) async def exec_test(turn_context: TurnContext) -> None: - dc = await dialogs.create_context(turn_context) + dialog_context = await dialogs.create_context(turn_context) - await dc.continue_dialog() + await dialog_context.continue_dialog() if not turn_context.responded: - await dc.begin_dialog(WaterfallDialog.__name__) + await dialog_context.begin_dialog(WaterfallDialog.__name__) await convo_state.save_changes(turn_context) adapter = TestAdapter(exec_test)