Axsuarez/blob storage

libraries/botbuilder-azure/botbuilder/azure/__init__.py

@@ -7,5 +7,13 @@
 
 from .about import __version__
 from .cosmosdb_storage import CosmosDbStorage, CosmosDbConfig, CosmosDbKeyEscape
+from .blob_storage import BlobStorage, BlobStorageSettings
 
-__all__ = ["CosmosDbStorage", "CosmosDbConfig", "CosmosDbKeyEscape", "__version__"]
+__all__ = [
+    "BlobStorage",
+    "BlobStorageSettings",
+    "CosmosDbStorage",
+    "CosmosDbConfig",
+    "CosmosDbKeyEscape",
+    "__version__",
+]

libraries/botbuilder-azure/botbuilder/azure/blob_storage.py

@@ -0,0 +1,102 @@
+import json
+from typing import Dict, List
+
+from azure.storage.blob import BlockBlobService, Blob, PublicAccess
+from botbuilder.core import Storage, StoreItem
+
+# TODO: sanitize_blob_name
+
+
+class BlobStorageSettings:
+    def __init__(
+        self,
+        container_name: str,
+        account_name: str = "",
+        account_key: str = "",
+        connection_string: str = "",
+    ):
+        self.container_name = container_name
+        self.account_name = account_name
+        self.account_key = account_key
+        self.connection_string = connection_string
+
+
+class BlobStorage(Storage):
+    def __init__(self, settings: BlobStorageSettings):
+        if settings.connection_string:
+            client = BlockBlobService(connection_string=settings.connection_string)
+        elif settings.account_name and settings.account_key:
+            client = BlockBlobService(
+                account_name=settings.account_name, account_key=settings.account_key
+            )
+        else:
+            raise Exception(
+                "Connection string should be provided if there are no account name and key"
+            )
+
+        self.client = client
+        self.settings = settings
+
+    async def read(self, keys: List[str]) -> Dict[str, object]:
+        if not keys:
+            raise Exception("Please provide at least one key to read from storage.")
+
+        self.client.create_container(self.settings.container_name)
+        self.client.set_container_acl(
+            self.settings.container_name, public_access=PublicAccess.Container
+        )
+        items = {}
+
+        for key in keys:
+            if self.client.exists(
+                container_name=self.settings.container_name, blob_name=key
+            ):
+                items[key] = self._blob_to_store_item(
+                    self.client.get_blob_to_text(
+                        container_name=self.settings.container_name, blob_name=key
+                    )
+                )
+
+        return items
+
+    async def write(self, changes: Dict[str, StoreItem]):
+        self.client.create_container(self.settings.container_name)
+        self.client.set_container_acl(
+            self.settings.container_name, public_access=PublicAccess.Container
+        )
+
+        for name, item in changes.items():
+            e_tag = (
+                None if not hasattr(item, "e_tag") or item.e_tag == "*" else item.e_tag
+            )
+            if e_tag:
+                item.e_tag = e_tag.replace('"', '\\"')
+            self.client.create_blob_from_text(
+                container_name=self.settings.container_name,
+                blob_name=name,
+                text=str(item),
+                if_match=e_tag,
+            )
+
+    async def delete(self, keys: List[str]):
+        if keys is None:
+            raise Exception("BlobStorage.delete: keys parameter can't be null")
+
+        self.client.create_container(self.settings.container_name)
+        self.client.set_container_acl(
+            self.settings.container_name, public_access=PublicAccess.Container
+        )
+
+        for key in keys:
+            if self.client.exists(
+                container_name=self.settings.container_name, blob_name=key
+            ):
+                self.client.delete_blob(
+                    container_name=self.settings.container_name, blob_name=key
+                )
+
+    def _blob_to_store_item(self, blob: Blob) -> StoreItem:
+        item = json.loads(blob.content)
+        item["e_tag"] = blob.properties.etag
+        item["id"] = blob.name
+        return StoreItem(**item)

libraries/botbuilder-azure/setup.py

@@ -6,6 +6,7 @@
 
 REQUIRES = [
     "azure-cosmos>=3.0.0",
+    "azure-storage-blob>=2.1.0",
     "botbuilder-schema>=4.4.0b1",
     "botframework-connector>=4.4.0b1",
 ]

libraries/botbuilder-azure/tests/test_blob_storage.py

@@ -0,0 +1,189 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import pytest
+from botbuilder.core import StoreItem
+from botbuilder.azure import BlobStorage, BlobStorageSettings
+
+# local blob emulator instance blob
+BLOB_STORAGE_SETTINGS = BlobStorageSettings(
+    account_name="", account_key="", container_name="test"
+)
+EMULATOR_RUNNING = False
+
+
+async def reset():
+    storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+    try:
+        await storage.client.delete_container(
+            container_name=BLOB_STORAGE_SETTINGS.container_name
+        )
+    except Exception:
+        pass
+
+
+class SimpleStoreItem(StoreItem):
+    def __init__(self, counter=1, e_tag="*"):
+        super(SimpleStoreItem, self).__init__()
+        self.counter = counter
+        self.e_tag = e_tag
+
+
+class TestBlobStorage:
+    @pytest.mark.asyncio
+    async def test_blob_storage_init_should_error_without_cosmos_db_config(self):
+        try:
+            BlobStorage(BlobStorageSettings())  # pylint: disable=no-value-for-parameter
+        except Exception as error:
+            assert error
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_read_should_return_data_with_valid_key(self):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"user": SimpleStoreItem()})
+
+        data = await storage.read(["user"])
+        assert "user" in data
+        assert data["user"].counter == "1"
+        assert len(data.keys()) == 1
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_read_update_should_return_new_etag(self):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"test": SimpleStoreItem(counter=1)})
+        data_result = await storage.read(["test"])
+        data_result["test"].counter = 2
+        await storage.write(data_result)
+        data_updated = await storage.read(["test"])
+        assert data_updated["test"].counter == "2"
+        assert data_updated["test"].e_tag != data_result["test"].e_tag
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_read_no_key_should_throw(self):
+        try:
+            storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+            await storage.read([])
+        except Exception as error:
+            assert error
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_write_should_add_new_value(self):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"user": SimpleStoreItem(counter=1)})
+
+        data = await storage.read(["user"])
+        assert "user" in data
+        assert data["user"].counter == "1"
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk(
+        self
+    ):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"user": SimpleStoreItem()})
+
+        await storage.write({"user": SimpleStoreItem(counter=10, e_tag="*")})
+        data = await storage.read(["user"])
+        assert data["user"].counter == "10"
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_write_batch_operation(self):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write(
+            {
+                "batch1": SimpleStoreItem(counter=1),
+                "batch2": SimpleStoreItem(counter=1),
+                "batch3": SimpleStoreItem(counter=1),
+            }
+        )
+        data = await storage.read(["batch1", "batch2", "batch3"])
+        assert len(data.keys()) == 3
+        assert data["batch1"]
+        assert data["batch2"]
+        assert data["batch3"]
+        assert data["batch1"].counter == "1"
+        assert data["batch2"].counter == "1"
+        assert data["batch3"].counter == "1"
+        assert data["batch1"].e_tag
+        assert data["batch2"].e_tag
+        assert data["batch3"].e_tag
+        await storage.delete(["batch1", "batch2", "batch3"])
+        data = await storage.read(["batch1", "batch2", "batch3"])
+        assert not data.keys()
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_delete_should_delete_according_cached_data(self):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"test": SimpleStoreItem()})
+        try:
+            await storage.delete(["test"])
+        except Exception as error:
+            raise error
+        else:
+            data = await storage.read(["test"])
+
+            assert isinstance(data, dict)
+            assert not data.keys()
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys(
+        self
+    ):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)})
+
+        await storage.delete(["test", "test2"])
+        data = await storage.read(["test", "test2"])
+        assert not data.keys()
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data(
+        self
+    ):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write(
+            {
+                "test": SimpleStoreItem(),
+                "test2": SimpleStoreItem(counter=2),
+                "test3": SimpleStoreItem(counter=3),
+            }
+        )
+
+        await storage.delete(["test", "test2"])
+        data = await storage.read(["test", "test2", "test3"])
+        assert len(data.keys()) == 1
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data(
+        self
+    ):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"test": SimpleStoreItem()})
+
+        await storage.delete(["foo"])
+        data = await storage.read(["test"])
+        assert len(data.keys()) == 1
+        data = await storage.read(["foo"])
+        assert not data.keys()
+
+    @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+    @pytest.mark.asyncio
+    async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data(
+        self
+    ):
+        storage = BlobStorage(BLOB_STORAGE_SETTINGS)
+        await storage.write({"test": SimpleStoreItem()})
+
+        await storage.delete(["foo", "bar"])
+        data = await storage.read(["test"])
+        assert len(data.keys()) == 1

libraries/botbuilder-core/botbuilder/core/storage.py

@@ -49,7 +49,7 @@ def __str__(self):
         output = (
             "{"
             + ",".join(
-                [f" '{attr}': '{getattr(self, attr)}'" for attr in non_magic_attributes]
+                [f' "{attr}": "{getattr(self, attr)}"' for attr in non_magic_attributes]
             )
             + " }"
         )