diff --git a/libraries/botbuilder-azure/botbuilder/azure/__init__.py b/libraries/botbuilder-azure/botbuilder/azure/__init__.py index 1f66a5f4f..54dea209d 100644 --- a/libraries/botbuilder-azure/botbuilder/azure/__init__.py +++ b/libraries/botbuilder-azure/botbuilder/azure/__init__.py @@ -7,5 +7,13 @@ from .about import __version__ from .cosmosdb_storage import CosmosDbStorage, CosmosDbConfig, CosmosDbKeyEscape +from .blob_storage import BlobStorage, BlobStorageSettings -__all__ = ["CosmosDbStorage", "CosmosDbConfig", "CosmosDbKeyEscape", "__version__"] +__all__ = [ + "BlobStorage", + "BlobStorageSettings", + "CosmosDbStorage", + "CosmosDbConfig", + "CosmosDbKeyEscape", + "__version__", +] diff --git a/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py new file mode 100644 index 000000000..f1c6eaf4d --- /dev/null +++ b/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py @@ -0,0 +1,102 @@ +import json +from typing import Dict, List + +from azure.storage.blob import BlockBlobService, Blob, PublicAccess +from botbuilder.core import Storage, StoreItem + +# TODO: sanitize_blob_name + + +class BlobStorageSettings: + def __init__( + self, + container_name: str, + account_name: str = "", + account_key: str = "", + connection_string: str = "", + ): + self.container_name = container_name + self.account_name = account_name + self.account_key = account_key + self.connection_string = connection_string + + +class BlobStorage(Storage): + def __init__(self, settings: BlobStorageSettings): + if settings.connection_string: + client = BlockBlobService(connection_string=settings.connection_string) + elif settings.account_name and settings.account_key: + client = BlockBlobService( + account_name=settings.account_name, account_key=settings.account_key + ) + else: + raise Exception( + "Connection string should be provided if there are no account name and key" + ) + + self.client = client + self.settings = settings + + async def read(self, keys: List[str]) -> Dict[str, object]: + if not keys: + raise Exception("Please provide at least one key to read from storage.") + + self.client.create_container(self.settings.container_name) + self.client.set_container_acl( + self.settings.container_name, public_access=PublicAccess.Container + ) + items = {} + + for key in keys: + if self.client.exists( + container_name=self.settings.container_name, blob_name=key + ): + items[key] = self._blob_to_store_item( + self.client.get_blob_to_text( + container_name=self.settings.container_name, blob_name=key + ) + ) + + return items + + async def write(self, changes: Dict[str, StoreItem]): + self.client.create_container(self.settings.container_name) + self.client.set_container_acl( + self.settings.container_name, public_access=PublicAccess.Container + ) + + for name, item in changes.items(): + e_tag = ( + None if not hasattr(item, "e_tag") or item.e_tag == "*" else item.e_tag + ) + if e_tag: + item.e_tag = e_tag.replace('"', '\\"') + self.client.create_blob_from_text( + container_name=self.settings.container_name, + blob_name=name, + text=str(item), + if_match=e_tag, + ) + + async def delete(self, keys: List[str]): + if keys is None: + raise Exception("BlobStorage.delete: keys parameter can't be null") + + self.client.create_container(self.settings.container_name) + self.client.set_container_acl( + self.settings.container_name, public_access=PublicAccess.Container + ) + + for key in keys: + if self.client.exists( + container_name=self.settings.container_name, blob_name=key + ): + self.client.delete_blob( + container_name=self.settings.container_name, blob_name=key + ) + + def _blob_to_store_item(self, blob: Blob) -> StoreItem: + item = json.loads(blob.content) + item["e_tag"] = blob.properties.etag + item["id"] = blob.name + return StoreItem(**item) diff --git a/libraries/botbuilder-azure/setup.py b/libraries/botbuilder-azure/setup.py index 2ebb15f3b..74c2ef81e 100644 --- a/libraries/botbuilder-azure/setup.py +++ b/libraries/botbuilder-azure/setup.py @@ -6,6 +6,7 @@ REQUIRES = [ "azure-cosmos>=3.0.0", + "azure-storage-blob>=2.1.0", "botbuilder-schema>=4.4.0b1", "botframework-connector>=4.4.0b1", ] diff --git a/libraries/botbuilder-azure/tests/test_blob_storage.py b/libraries/botbuilder-azure/tests/test_blob_storage.py new file mode 100644 index 000000000..4ccaf1225 --- /dev/null +++ b/libraries/botbuilder-azure/tests/test_blob_storage.py @@ -0,0 +1,189 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from botbuilder.core import StoreItem +from botbuilder.azure import BlobStorage, BlobStorageSettings + +# local blob emulator instance blob +BLOB_STORAGE_SETTINGS = BlobStorageSettings( + account_name="", account_key="", container_name="test" +) +EMULATOR_RUNNING = False + + +async def reset(): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + try: + await storage.client.delete_container( + container_name=BLOB_STORAGE_SETTINGS.container_name + ) + except Exception: + pass + + +class SimpleStoreItem(StoreItem): + def __init__(self, counter=1, e_tag="*"): + super(SimpleStoreItem, self).__init__() + self.counter = counter + self.e_tag = e_tag + + +class TestBlobStorage: + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_without_cosmos_db_config(self): + try: + BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter + except Exception as error: + assert error + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_read_should_return_data_with_valid_key(self): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"user": SimpleStoreItem()}) + + data = await storage.read(["user"]) + assert "user" in data + assert data["user"].counter == "1" + assert len(data.keys()) == 1 + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_read_update_should_return_new_etag(self): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"test": SimpleStoreItem(counter=1)}) + data_result = await storage.read(["test"]) + data_result["test"].counter = 2 + await storage.write(data_result) + data_updated = await storage.read(["test"]) + assert data_updated["test"].counter == "2" + assert data_updated["test"].e_tag != data_result["test"].e_tag + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_read_no_key_should_throw(self): + try: + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.read([]) + except Exception as error: + assert error + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_write_should_add_new_value(self): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"user": SimpleStoreItem(counter=1)}) + + data = await storage.read(["user"]) + assert "user" in data + assert data["user"].counter == "1" + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk( + self + ): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"user": SimpleStoreItem()}) + + await storage.write({"user": SimpleStoreItem(counter=10, e_tag="*")}) + data = await storage.read(["user"]) + assert data["user"].counter == "10" + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_write_batch_operation(self): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write( + { + "batch1": SimpleStoreItem(counter=1), + "batch2": SimpleStoreItem(counter=1), + "batch3": SimpleStoreItem(counter=1), + } + ) + data = await storage.read(["batch1", "batch2", "batch3"]) + assert len(data.keys()) == 3 + assert data["batch1"] + assert data["batch2"] + assert data["batch3"] + assert data["batch1"].counter == "1" + assert data["batch2"].counter == "1" + assert data["batch3"].counter == "1" + assert data["batch1"].e_tag + assert data["batch2"].e_tag + assert data["batch3"].e_tag + await storage.delete(["batch1", "batch2", "batch3"]) + data = await storage.read(["batch1", "batch2", "batch3"]) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_according_cached_data(self): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"test": SimpleStoreItem()}) + try: + await storage.delete(["test"]) + except Exception as error: + raise error + else: + data = await storage.read(["test"]) + + assert isinstance(data, dict) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( + self + ): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)}) + + await storage.delete(["test", "test2"]) + data = await storage.read(["test", "test2"]) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data( + self + ): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write( + { + "test": SimpleStoreItem(), + "test2": SimpleStoreItem(counter=2), + "test3": SimpleStoreItem(counter=3), + } + ) + + await storage.delete(["test", "test2"]) + data = await storage.read(["test", "test2", "test3"]) + assert len(data.keys()) == 1 + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data( + self + ): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"test": SimpleStoreItem()}) + + await storage.delete(["foo"]) + data = await storage.read(["test"]) + assert len(data.keys()) == 1 + data = await storage.read(["foo"]) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data( + self + ): + storage = BlobStorage(BLOB_STORAGE_SETTINGS) + await storage.write({"test": SimpleStoreItem()}) + + await storage.delete(["foo", "bar"]) + data = await storage.read(["test"]) + assert len(data.keys()) == 1 diff --git a/libraries/botbuilder-core/botbuilder/core/storage.py b/libraries/botbuilder-core/botbuilder/core/storage.py index 0d5b50029..8c9a1f1ab 100644 --- a/libraries/botbuilder-core/botbuilder/core/storage.py +++ b/libraries/botbuilder-core/botbuilder/core/storage.py @@ -49,7 +49,7 @@ def __str__(self): output = ( "{" + ",".join( - [f" '{attr}': '{getattr(self, attr)}'" for attr in non_magic_attributes] + [f' "{attr}": "{getattr(self, attr)}"' for attr in non_magic_attributes] ) + " }" )