increase test coverage, add CACHE_FILES_PATH to settings

Author: MHHukiewitz

src/aleph/sdk/cache.py

@@ -290,7 +290,7 @@ async def get_posts(
         conditions = []
 
         if types:
-            conditions.append(query_post_types(types))
+            conditions.append(query_message_types(types))
         if refs:
             conditions.append(query_refs(refs))
         if addresses:
@@ -345,7 +345,7 @@ async def get_messages(
         conditions = []
 
         if message_type:
-            conditions.append(MessageModel.type == message_type.value)
+            conditions.append(query_message_types(message_type))
         if content_types:
             conditions.append(query_content_types(content_types))
         if content_keys:
@@ -455,10 +455,10 @@ async def watch_messages(
             yield model_to_message(item)
 
 
-def query_post_types(types: Union[str, Iterable[str]]):
-    if isinstance(types, str):
-        return MessageModel.content_type == types
-    return MessageModel.content_type.in_(types)
+def query_message_types(message_types: Union[str, Iterable[str]]):
+    if isinstance(message_types, str):
+        return MessageModel.type == message_types
+    return MessageModel.type.in_(message_types)
 
 
 def query_content_types(content_types: Union[str, Iterable[str]]):

src/aleph/sdk/conf.py

@@ -37,6 +37,10 @@ class Settings(BaseSettings):
         default=Path(":memory:"),  # can also be :memory: for in-memory caching
         description="Path to the cache database",
     )
+    CACHE_FILES_PATH: Path = Field(
+        default=Path("cache", "files"),
+        description="Path to the cache files",
+    )
 
     class Config:
         env_prefix = "ALEPH_"

src/aleph/sdk/node.py

@@ -16,6 +16,7 @@
 from aleph.sdk import AuthenticatedAlephClient
 from aleph.sdk.base import AuthenticatedAlephClientBase
 from aleph.sdk.cache import MessageCache
+from aleph.sdk.conf import settings
 from aleph.sdk.types import StorageEnum
 
 
@@ -107,13 +108,14 @@ async def download_file(self, file_hash: str) -> bytes:
                 return f.read()
         except FileNotFoundError:
             file = await self.session.download_file(file_hash)
+            self._file_path(file_hash).parent.mkdir(parents=True, exist_ok=True)
             with open(self._file_path(file_hash), "wb") as f:
                 f.write(file)
             return file
 
-    def _file_path(self, file_hash: str) -> Path:
-        # TODO: Make this configurable (and not be an ugly hack)
-        return Path("cache", "files", file_hash)
+    @staticmethod
+    def _file_path(file_hash: str) -> Path:
+        return settings.CACHE_FILES_PATH / Path(file_hash)
 
     async def create_post(
         self,
@@ -271,5 +273,5 @@ async def submit(
         )
         # TODO: this can cause inconsistencies if the message is dropped
         if status in [MessageStatus.PROCESSED, MessageStatus.PENDING]:
-            self.add(resp["message"])
+            self.add(resp)
         return resp, status

tests/unit/conftest.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 from tempfile import NamedTemporaryFile
+from typing import List
 
 import pytest as pytest
+from aleph_message.models import AggregateMessage, AlephMessage, PostMessage
 
 import aleph.sdk.chains.ethereum as ethereum
 import aleph.sdk.chains.sol as solana
@@ -34,3 +36,71 @@ def tezos_account() -> tezos.TezosAccount:
     with NamedTemporaryFile(delete=False) as private_key_file:
         private_key_file.close()
         yield tezos.get_fallback_account(path=Path(private_key_file.name))
+
+
+@pytest.fixture
+def messages() -> List[AlephMessage]:
+    return [
+        AggregateMessage.parse_obj(
+            {
+                "item_hash": "5b26d949fe05e38f535ef990a89da0473f9d700077cced228f2d36e73fca1fd6",
+                "type": "AGGREGATE",
+                "chain": "ETH",
+                "sender": "0x51A58800b26AA1451aaA803d1746687cB88E0501",
+                "signature": "0xca5825b6b93390482b436cb7f28b4628f8c9f56dc6af08260c869b79dd6017c94248839bd9fd0ffa1230dc3b1f4f7572a8d1f6fed6c6e1fb4d70ccda0ab5d4f21b",
+                "item_type": "inline",
+                "item_content": '{"address":"0x51A58800b26AA1451aaA803d1746687cB88E0501","key":"0xce844d79e5c0c325490c530aa41e8f602f0b5999binance","content":{"1692026263168":{"version":"x25519-xsalsa20-poly1305","nonce":"RT4Lbqs7Xzk+op2XC+VpXgwOgg21BotN","ephemPublicKey":"CVW8ECE3m8BepytHMTLan6/jgIfCxGdnKmX47YirF08=","ciphertext":"VuGJ9vMkJSbaYZCCv6Zemx4ixeb+9IW8H1vFB9vLtz1a8d87R4BfYUisLoCQxRkeUXqfW0/KIGQ5idVjr8Yj7QnKglW5AJ8UX7wEWMhiRFLatpWP8P9FI2n8Z7Rblu7Oz/OeKnuljKL3KsalcUQSsFa/1qACsIoycPZ6Wq6t1mXxVxxJWzClLyKRihv1pokZGT9UWxh7+tpoMGlRdYainyAt0/RygFw+r8iCMOilHnyv4ndLkKQJXyttb0tdNr/gr57+9761+trioGSysLQKZQWW6Ih6aE8V9t3BenfzYwiCnfFw3YAAKBPMdm9QdIETyrOi7YhD/w==","sha256":"bbeb499f681aed2bc18b6f3b6a30d25254bd30fbfde43444e9085f3bcd075c3c"}},"time":1692026263.662}',
+                "content": {
+                    "key": "0xce844d79e5c0c325490c530aa41e8f602f0b5999binance",
+                    "time": 1692026263.662,
+                    "address": "0x51A58800b26AA1451aaA803d1746687cB88E0501",
+                    "content": {
+                        "hello": "world",
+                    },
+                },
+                "time": 1692026263.662,
+                "channel": "UNSLASHED",
+                "size": 734,
+                "confirmations": [],
+                "confirmed": False,
+            }
+        ),
+        PostMessage.parse_obj(
+            {
+                "item_hash": "70f3798fdc68ce0ee03715a5547ee24e2c3e259bf02e3f5d1e4bf5a6f6a5e99f",
+                "type": "POST",
+                "chain": "SOL",
+                "sender": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4",
+                "signature": "0x91616ee45cfba55742954ff87ebf86db4988bcc5e3334b49a4caa6436e28e28d4ab38667cbd4bfb8903abf8d71f70d9ceb2c0a8d0a15c04fc1af5657f0050c101b",
+                "item_type": "storage",
+                "item_content": None,
+                "content": {
+                    "time": 1692026021.1257718,
+                    "type": "aleph-network-metrics",
+                    "address": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4",
+                    "ref": "0123456789abcdef",
+                    "content": {
+                        "tags": ["mainnet"],
+                        "hello": "world",
+                        "version": "1.0",
+                    },
+                },
+                "time": 1692026021.132849,
+                "channel": "aleph-scoring",
+                "size": 122537,
+                "confirmations": [],
+                "confirmed": False,
+            }
+        ),
+    ]
+
+
+@pytest.fixture
+def raw_messages_response(messages):
+    return {
+        "messages": [message.dict() for message in messages],
+        "pagination_item": "messages",
+        "pagination_page": 1,
+        "pagination_per_page": 20,
+        "pagination_total": 2,
+    }

tests/unit/test_cache.py

@@ -4,7 +4,6 @@
 
 import pytest
 from aleph_message.models import (
-    AggregateMessage,
     AlephMessage,
     Chain,
     MessageType,
@@ -16,63 +15,6 @@
 from aleph.sdk.chains.ethereum import get_fallback_account
 
 
-@pytest.fixture(scope="session")
-def messages():
-    return [
-        AggregateMessage.parse_obj(
-            {
-                "item_hash": "5b26d949fe05e38f535ef990a89da0473f9d700077cced228f2d36e73fca1fd6",
-                "type": "AGGREGATE",
-                "chain": "ETH",
-                "sender": "0x51A58800b26AA1451aaA803d1746687cB88E0501",
-                "signature": "0xca5825b6b93390482b436cb7f28b4628f8c9f56dc6af08260c869b79dd6017c94248839bd9fd0ffa1230dc3b1f4f7572a8d1f6fed6c6e1fb4d70ccda0ab5d4f21b",
-                "item_type": "inline",
-                "item_content": '{"address":"0x51A58800b26AA1451aaA803d1746687cB88E0501","key":"0xce844d79e5c0c325490c530aa41e8f602f0b5999binance","content":{"1692026263168":{"version":"x25519-xsalsa20-poly1305","nonce":"RT4Lbqs7Xzk+op2XC+VpXgwOgg21BotN","ephemPublicKey":"CVW8ECE3m8BepytHMTLan6/jgIfCxGdnKmX47YirF08=","ciphertext":"VuGJ9vMkJSbaYZCCv6Zemx4ixeb+9IW8H1vFB9vLtz1a8d87R4BfYUisLoCQxRkeUXqfW0/KIGQ5idVjr8Yj7QnKglW5AJ8UX7wEWMhiRFLatpWP8P9FI2n8Z7Rblu7Oz/OeKnuljKL3KsalcUQSsFa/1qACsIoycPZ6Wq6t1mXxVxxJWzClLyKRihv1pokZGT9UWxh7+tpoMGlRdYainyAt0/RygFw+r8iCMOilHnyv4ndLkKQJXyttb0tdNr/gr57+9761+trioGSysLQKZQWW6Ih6aE8V9t3BenfzYwiCnfFw3YAAKBPMdm9QdIETyrOi7YhD/w==","sha256":"bbeb499f681aed2bc18b6f3b6a30d25254bd30fbfde43444e9085f3bcd075c3c"}},"time":1692026263.662}',
-                "content": {
-                    "key": "0xce844d79e5c0c325490c530aa41e8f602f0b5999binance",
-                    "time": 1692026263.662,
-                    "address": "0x51A58800b26AA1451aaA803d1746687cB88E0501",
-                    "content": {
-                        "hello": "world",
-                    },
-                },
-                "time": 1692026263.662,
-                "channel": "UNSLASHED",
-                "size": 734,
-                "confirmations": [],
-                "confirmed": False,
-            }
-        ),
-        PostMessage.parse_obj(
-            {
-                "item_hash": "70f3798fdc68ce0ee03715a5547ee24e2c3e259bf02e3f5d1e4bf5a6f6a5e99f",
-                "type": "POST",
-                "chain": "SOL",
-                "sender": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4",
-                "signature": "0x91616ee45cfba55742954ff87ebf86db4988bcc5e3334b49a4caa6436e28e28d4ab38667cbd4bfb8903abf8d71f70d9ceb2c0a8d0a15c04fc1af5657f0050c101b",
-                "item_type": "storage",
-                "item_content": None,
-                "content": {
-                    "time": 1692026021.1257718,
-                    "type": "aleph-network-metrics",
-                    "address": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4",
-                    "ref": "0123456789abcdef",
-                    "content": {
-                        "tags": ["mainnet"],
-                        "hello": "world",
-                        "version": "1.0",
-                    },
-                },
-                "time": 1692026021.132849,
-                "channel": "aleph-scoring",
-                "size": 122537,
-                "confirmations": [],
-                "confirmed": False,
-            }
-        ),
-    ]
-
-
 @pytest.mark.asyncio
 async def test_base(messages):
     # test add_many
@@ -171,6 +113,12 @@ async def test_chains(self):
             0
         ] == self.messages[1]
 
+    @pytest.mark.asyncio
+    async def test_content_keys(self):
+        assert (
+            await self.cache.get_messages(content_keys=self.messages[0].content.key)
+        ).messages[0] == self.messages[0]
+
 
 @pytest.mark.asyncio
 async def test_message_cache_listener():