From e0d201a9d91b53cb90c4baefa1319f7938e042e7 Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 21 Aug 2023 15:21:07 +0200 Subject: [PATCH 01/36] Features: Secure Endpoint --- src/aleph/storage.py | 5 +- tests/api/test_storage.py | 119 ++++++++++++++++++++++++++++++++++---- 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/src/aleph/storage.py b/src/aleph/storage.py index 39352d85d..d5e73a613 100644 --- a/src/aleph/storage.py +++ b/src/aleph/storage.py @@ -22,7 +22,9 @@ from aleph.types.db_session import DbSession from aleph.types.files import FileType from aleph.utils import get_sha256 - +from aleph.schemas.pending_messages import ( + parse_message, +) LOGGER = logging.getLogger(__name__) @@ -271,7 +273,6 @@ async def add_file( elif engine == ItemType.storage: file_content = fileobject.read() file_hash = sha256(file_content).hexdigest() - else: raise ValueError(f"Unsupported item type: {engine}") diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 0965186fb..f3e165bb6 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -1,16 +1,31 @@ import base64 +import datetime +import json +from hashlib import sha256 from typing import Any import aiohttp import orjson import pytest -from aleph_message.models import ItemHash +from aleph_message.models import ItemHash, MessageType, Chain, ItemType +from configmanager import Config + +from aleph.handlers.message_handler import MessageHandler +from aleph.schemas.pending_messages import ( + parse_message, +) +from decimal import Decimal from aleph.db.accessors.files import get_file +from aleph.db.models import PendingMessageDb, AlephBalanceDb +from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage from aleph.storage import StorageService +from aleph.toolkit.timestamp import timestamp_to_datetime +from aleph.types.channel import Channel from aleph.types.db_session import DbSessionFactory from aleph.types.files import FileType from in_memory_storage_engine import InMemoryStorageEngine +import datetime as dt IPFS_ADD_FILE_URI = "/api/v0/ipfs/add_file" IPFS_ADD_JSON_URI = "/api/v0/ipfs/add_json" @@ -63,11 +78,11 @@ def api_client(ccn_api_client, mocker): async def add_file( - api_client, - session_factory: DbSessionFactory, - uri: str, - file_content: bytes, - expected_file_hash: str, + api_client, + session_factory: DbSessionFactory, + uri: str, + file_content: bytes, + expected_file_hash: str, ): form_data = aiohttp.FormData() form_data.add_field("file", file_content) @@ -95,8 +110,79 @@ async def add_file( assert response_data == file_content + +async def add_file_with_message( + api_client, + session_factory: DbSessionFactory, + uri: str, + file_content: bytes, + expected_file_hash: str, +): + + data = { + "item_hash": "bb6e53f2738e5934b9a2125a9dc3d76211720e5152bdbcd4b236363d18d4f8a3", + "type": "STORE", + "chain": "ETH", + "sender": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106", + "item_content":'{"address": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106", "time": 1665478676.6585264, "item_type": "storage", "item_hash": "2086c8b69830df060f49bdf03a89e508688db7f5e5387bb875a6a0ed2d7a1d63", "mime_type": "text/plain"}', + "signature": "0xb9d164e6e43a8fcd341abc01eda47bed0333eaf480e888f2ed2ae0017048939d18850a33352e7281645e95e8673bad733499b6a8ce4069b9da9b9a79ddc1a0b31b", + "item_type": ItemType.storage, + "time": str(1616021679.055), + "channel": "TEST", + "trusted": "True", + } + test = { + "chain": "ETH", + "sender": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + "type": "STORE", + "channel": "null", + "signature": "0x2b90dcfa8f93506150df275a4fe670e826be0b4b751badd6ec323648a6a738962f47274f71a9939653fb6d49c25055821f547447fb3b33984a579008d93eca431b", + "time": "1692193373.7144432", + "item_type": "inline", + "item_content": "{\"address\":\"0x6dA130FD646f826C1b8080C07448923DF9a79aaA\",\"time\":1692193373.714271,\"item_type\":\"storage\",\"item_hash\":\"0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f\",\"mime_type\":\"text/plain\"}", + "item_hash": "8227acbc2f7c43899efd9f63ea9d8119a4cb142f3ba2db5fe499ccfab86dfaed", + "content": { + "address": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + "time": "1692193373.714271", + "item_type": "storage", + "item_hash": "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "mime_type": "text/plain" + }, + } + data["time"] = str(data["time"]) + + json_data = json.dumps(test) + form_data = aiohttp.FormData() + actual_item_hash = sha256(file_content).hexdigest() + form_data.add_field("file", file_content) + form_data.add_field("message", json_data, content_type='application/json') + form_data.add_field("size", str(len(file_content))) + post_response = await api_client.post(uri, data=form_data) + assert post_response.status == 200, await post_response.text() + #post_response_json = await post_response.json() + # data["time"] = 1644409598.782 + # pending_message = PendingMessageDb.from_message_dict( + # data, fetched=True, reception_time=dt.datetime(2022, 1, 1) + +def test_storage_add_file_with_message_only_parse(): + message_dict = { + "chain": "ETH", + "item_hash": "30cc40533aa3ccf16a7c7c8a40da5633f64a83e4b89dcc7815f3a0af2149e1ac", + "sender": "0x7332eA1229c11C627C10eB24c1A6F77BceD1D5c1", + "type": "STORE", + "channel": "EVIDENZ", + "item_content": None, + "item_type": "storage", + "signature": "23d1d099dd111ae3251efea537f57767cf43b2ae3611bf9051760e0a9bc2bd4429563a130e3e391668086d101f8a197f55377f50b15d4c0303ff957d90a258a31b", + "time": 1616021679.055, + } + + message = parse_message(message_dict) + + @pytest.mark.asyncio async def test_storage_add_file(api_client, session_factory: DbSessionFactory): + await add_file( api_client, session_factory, @@ -106,6 +192,17 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): ) +@pytest.mark.asyncio +async def test_storage_add_file_with_message(api_client, session_factory: DbSessionFactory): + await add_file_with_message( + api_client, + session_factory, + uri=STORAGE_ADD_FILE_URI, + file_content=b"Hello Aleph.im\n", + expected_file_hash="c25b0525bc308797d3e35763faf5c560f2974dab802cb4a734ae4e9d1040319e", + ) + + @pytest.mark.asyncio async def test_ipfs_add_file(api_client, session_factory: DbSessionFactory): await add_file( @@ -118,11 +215,11 @@ async def test_ipfs_add_file(api_client, session_factory: DbSessionFactory): async def add_json( - api_client, - session_factory: DbSessionFactory, - uri: str, - json: Any, - expected_file_hash: ItemHash, + api_client, + session_factory: DbSessionFactory, + uri: str, + json: Any, + expected_file_hash: ItemHash, ): serialized_json = orjson.dumps(json) From d96ffa40642a8bb69168bc2764578c360bd47639 Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 21 Aug 2023 15:21:07 +0200 Subject: [PATCH 02/36] Features: Secure upload endpoint --- src/aleph/web/controllers/storage.py | 129 +++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 5 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 8075ba2da..e265889d5 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -1,21 +1,49 @@ +import asyncio import base64 +import datetime as dt +import functools import logging +from hashlib import sha256 +from io import StringIO +from typing import Union, Tuple, Dict, Optional + +import aio_pika +from eth_account import Account +from eth_account.messages import encode_defunct + +from aleph.chains.common import get_verification_buffer +from aleph.jobs.process_pending_messages import PendingMessageProcessor from aiohttp import web +from aiohttp.web_request import FileField from aleph_message.models import ItemType +from multidict import MultiDictProxy +from aleph.chains.chain_service import ChainService, LOGGER +from aleph.chains.nuls import NulsConnector +from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.files import count_file_pins, get_file +from aleph.db.accessors.messages import get_message_status, message_exists +from aleph.db.connection import make_session_factory +from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError +from aleph.services.p2p import init_p2p_client +from aleph.services.storage.engine import StorageEngine +from aleph.services.storage.fileystem_engine import FileSystemStorageEngine +from aleph.storage import StorageService +from aleph.toolkit import json +from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.db_session import DbSessionFactory, DbSession from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, - get_storage_service_from_request, + get_storage_service_from_request, get_mq_channel_from_request, get_config_from_request, get_mq_conn_from_request, ) from aleph.web.controllers.utils import multidict_proxy_to_io +from aleph.schemas.pending_messages import BasePendingMessage logger = logging.getLogger(__name__) - +from aleph.schemas.pending_messages import parse_message MAX_FILE_SIZE = 100 * 1024 * 1024 @@ -56,20 +84,111 @@ async def add_storage_json_controller(request: web.Request): return web.json_response(output) -async def storage_add_file(request: web.Request): +async def verify_signature(message: BasePendingMessage) -> bool: + """Verifies a signature of a message, return True if verified, false if not""" + verification = get_verification_buffer(message) + + message_hash = await run_in_executor( + None, functools.partial(encode_defunct, text=verification.decode("utf-8")) + ) + + verified = False + try: + # we assume the signature is a valid string + address = await run_in_executor( + None, + functools.partial( + Account.recover_message, message_hash, signature=message.signature + ), + ) + if address == message.sender: + verified = True + else: + return False + + except Exception as e: + verified = False + return verified + + +async def get_message_content(post_data: MultiDictProxy[Union[str, bytes, FileField]]) -> Tuple[dict, int]: + message_bytearray = post_data.get("message", b"") + value = post_data.get("size") or 0 + if not message_bytearray: + return {}, int(value) # Empty dictionary if no message content + + message_string = message_bytearray.decode("utf-8") + message_dict = json.loads(message_string) + message_dict["time"] = float(message_dict["time"]) + + return message_dict, int(value) + + +async def init_mq_con(config): + return await aio_pika.connect_robust( + host=config.p2p.mq_host.value, port=config.rabbitmq.port.value, login=config.rabbitmq.username.value, + password=config.rabbitmq.password.value + ) + + +async def verify_and_handle_request(pending_message_db, file_io, message, size): + content = file_io.read(size) + item_content = json.loads(message["item_content"]) + actual_item_hash = sha256(content).hexdigest() + c_item_hash = item_content["item_hash"] + + is_signature = await verify_signature(message=pending_message_db) + + if not is_signature: + output = {"status": "Forbidden"} + return web.json_response(output, status=403) + elif actual_item_hash != c_item_hash: + output = {"status": "Unprocessable Content"} + return web.json_response(output, status=422) + elif len(content) > 25_000 and not message: + output = {"status": "Unauthorized"} + return web.json_response(output, status=401) + else: + return None + + +async def storage_add_file_with_message(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) + config = get_config_from_request(request) + mq_con = init_mq_con(config) - # No need to pin it here anymore. - # TODO: find a way to specify linked ipfs hashes in posts/aggr. post = await request.post() file_io = multidict_proxy_to_io(post) + message, size = await get_message_content(post) + pending_message_db = PendingMessageDb.from_message_dict(message_dict=message, reception_time=dt.datetime.now(), + fetched=True) + is_valid_message = await verify_and_handle_request(pending_message_db, file_io, message, size) + if is_valid_message is not None: + return is_valid_message + with session_factory() as session: file_hash = await storage_service.add_file( session=session, fileobject=file_io, engine=ItemType.storage ) + session.add(pending_message_db) session.commit() + output = {"status": "success", "hash": file_hash} + return web.json_response(output) + + +async def storage_add_file(request: web.Request): + post = await request.post() + if post.get("message", b"") is not None and post.get("size") is not None: + return await storage_add_file_with_message(request) + storage_service = get_storage_service_from_request(request) + session_factory = get_session_factory_from_request(request) + file_io = multidict_proxy_to_io(post) + with session_factory() as session: + file_hash = await storage_service.add_file( + session=session, fileobject=file_io, engine=ItemType.storage + ) output = {"status": "success", "hash": file_hash} return web.json_response(output) From cccb53f46199b2ba8b560c8e0425e841e5a2a72f Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 21 Aug 2023 15:37:40 +0200 Subject: [PATCH 03/36] Fix : Clear test --- tests/api/test_storage.py | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index f3e165bb6..bd3a05992 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -118,20 +118,7 @@ async def add_file_with_message( file_content: bytes, expected_file_hash: str, ): - data = { - "item_hash": "bb6e53f2738e5934b9a2125a9dc3d76211720e5152bdbcd4b236363d18d4f8a3", - "type": "STORE", - "chain": "ETH", - "sender": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106", - "item_content":'{"address": "0x696879aE4F6d8DaDD5b8F1cbb1e663B89b08f106", "time": 1665478676.6585264, "item_type": "storage", "item_hash": "2086c8b69830df060f49bdf03a89e508688db7f5e5387bb875a6a0ed2d7a1d63", "mime_type": "text/plain"}', - "signature": "0xb9d164e6e43a8fcd341abc01eda47bed0333eaf480e888f2ed2ae0017048939d18850a33352e7281645e95e8673bad733499b6a8ce4069b9da9b9a79ddc1a0b31b", - "item_type": ItemType.storage, - "time": str(1616021679.055), - "channel": "TEST", - "trusted": "True", - } - test = { "chain": "ETH", "sender": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", "type": "STORE", @@ -149,35 +136,16 @@ async def add_file_with_message( "mime_type": "text/plain" }, } - data["time"] = str(data["time"]) - json_data = json.dumps(test) + json_data = json.dumps(data) form_data = aiohttp.FormData() - actual_item_hash = sha256(file_content).hexdigest() + form_data.add_field("file", file_content) form_data.add_field("message", json_data, content_type='application/json') form_data.add_field("size", str(len(file_content))) post_response = await api_client.post(uri, data=form_data) assert post_response.status == 200, await post_response.text() #post_response_json = await post_response.json() - # data["time"] = 1644409598.782 - # pending_message = PendingMessageDb.from_message_dict( - # data, fetched=True, reception_time=dt.datetime(2022, 1, 1) - -def test_storage_add_file_with_message_only_parse(): - message_dict = { - "chain": "ETH", - "item_hash": "30cc40533aa3ccf16a7c7c8a40da5633f64a83e4b89dcc7815f3a0af2149e1ac", - "sender": "0x7332eA1229c11C627C10eB24c1A6F77BceD1D5c1", - "type": "STORE", - "channel": "EVIDENZ", - "item_content": None, - "item_type": "storage", - "signature": "23d1d099dd111ae3251efea537f57767cf43b2ae3611bf9051760e0a9bc2bd4429563a130e3e391668086d101f8a197f55377f50b15d4c0303ff957d90a258a31b", - "time": 1616021679.055, - } - - message = parse_message(message_dict) @pytest.mark.asyncio From 3873b96ec87266c1a1a303a783ac31a4f2899210 Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 12:45:10 +0200 Subject: [PATCH 04/36] Internal: add test for /add_file (200, 422, 402 tested) --- tests/api/test_storage.py | 139 +++++++++++++++++++++++++------------- 1 file changed, 93 insertions(+), 46 deletions(-) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index bd3a05992..767f9ea97 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -46,6 +46,25 @@ "b7c7b2db0bcec890b8c859b2b76e7c998de15e31ccc945bc7425c4bdc091a0b2" ) +MESSAGE_DICT = { + "chain": "ETH", + "sender": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + "type": "STORE", + "channel": "null", + "signature": "0x2b90dcfa8f93506150df275a4fe670e826be0b4b751badd6ec323648a6a738962f47274f71a9939653fb6d49c25055821f547447fb3b33984a579008d93eca431b", + "time": "1692193373.7144432", + "item_type": "inline", + "item_content": '{"address":"0x6dA130FD646f826C1b8080C07448923DF9a79aaA","time":1692193373.714271,"item_type":"storage","item_hash":"0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f","mime_type":"text/plain"}', + "item_hash": "8227acbc2f7c43899efd9f63ea9d8119a4cb142f3ba2db5fe499ccfab86dfaed", + "content": { + "address": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + "time": "1692193373.714271", + "item_type": "storage", + "item_hash": "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "mime_type": "text/plain", + }, +} + @pytest.fixture def api_client(ccn_api_client, mocker): @@ -74,20 +93,21 @@ def api_client(ccn_api_client, mocker): ipfs_service=ipfs_service, node_cache=mocker.AsyncMock(), ) + return ccn_api_client async def add_file( - api_client, - session_factory: DbSessionFactory, - uri: str, - file_content: bytes, - expected_file_hash: str, + api_client, + session_factory: DbSessionFactory, + uri: str, + file_content: bytes, + expected_file_hash: str, ): form_data = aiohttp.FormData() form_data.add_field("file", file_content) - post_response = await api_client.post(uri, data=form_data) + post_response = await api_client.post(uri, data=form_data, sync=True) assert post_response.status == 200, await post_response.text() post_response_json = await post_response.json() assert post_response_json["status"] == "success" @@ -110,47 +130,38 @@ async def add_file( assert response_data == file_content - async def add_file_with_message( - api_client, - session_factory: DbSessionFactory, - uri: str, - file_content: bytes, - expected_file_hash: str, + api_client, + session_factory: DbSessionFactory, + uri: str, + file_content: bytes, + size: str, + error_code: int, + balance: int, ): - data = { - "chain": "ETH", - "sender": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", - "type": "STORE", - "channel": "null", - "signature": "0x2b90dcfa8f93506150df275a4fe670e826be0b4b751badd6ec323648a6a738962f47274f71a9939653fb6d49c25055821f547447fb3b33984a579008d93eca431b", - "time": "1692193373.7144432", - "item_type": "inline", - "item_content": "{\"address\":\"0x6dA130FD646f826C1b8080C07448923DF9a79aaA\",\"time\":1692193373.714271,\"item_type\":\"storage\",\"item_hash\":\"0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f\",\"mime_type\":\"text/plain\"}", - "item_hash": "8227acbc2f7c43899efd9f63ea9d8119a4cb142f3ba2db5fe499ccfab86dfaed", - "content": { - "address": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", - "time": "1692193373.714271", - "item_type": "storage", - "item_hash": "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - "mime_type": "text/plain" - }, - } - - json_data = json.dumps(data) + with session_factory() as session: + session.add( + AlephBalanceDb( + address="0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + chain=Chain.ETH, + balance=Decimal(balance), + eth_height=0, + ) + ) + session.commit() + + json_data = json.dumps(MESSAGE_DICT) form_data = aiohttp.FormData() form_data.add_field("file", file_content) - form_data.add_field("message", json_data, content_type='application/json') - form_data.add_field("size", str(len(file_content))) - post_response = await api_client.post(uri, data=form_data) - assert post_response.status == 200, await post_response.text() - #post_response_json = await post_response.json() + form_data.add_field("message", json_data, content_type="application/json") + form_data.add_field("size", size) + response = await api_client.post(uri, data=form_data) + assert response.status == error_code, await response.text() @pytest.mark.asyncio async def test_storage_add_file(api_client, session_factory: DbSessionFactory): - await add_file( api_client, session_factory, @@ -160,14 +171,50 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): ) +@pytest.mark.parametrize( + "file_content, expected_hash, size, error_code, balance", + [ + ( + b"Hello Aleph.im\n", + "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "15", + "402", + "0", + ), + ( + b"Hello Aleph.im\n", + "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "10", + "422", + "1000", + ), + ( + b"Hello Aleph.im\n", + "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "15", + "200", + "1000", + ), + ], +) @pytest.mark.asyncio -async def test_storage_add_file_with_message(api_client, session_factory: DbSessionFactory): +async def test_storage_add_file_with_message( + api_client, + session_factory: DbSessionFactory, + file_content, + expected_hash, + size, + error_code, + balance, +): await add_file_with_message( api_client, session_factory, uri=STORAGE_ADD_FILE_URI, - file_content=b"Hello Aleph.im\n", - expected_file_hash="c25b0525bc308797d3e35763faf5c560f2974dab802cb4a734ae4e9d1040319e", + file_content=file_content, + size=size, + error_code=int(error_code), + balance=int(balance), ) @@ -183,11 +230,11 @@ async def test_ipfs_add_file(api_client, session_factory: DbSessionFactory): async def add_json( - api_client, - session_factory: DbSessionFactory, - uri: str, - json: Any, - expected_file_hash: ItemHash, + api_client, + session_factory: DbSessionFactory, + uri: str, + json: Any, + expected_file_hash: ItemHash, ): serialized_json = orjson.dumps(json) From 73675094005e533bb4619d7b572f7b997a7c93b5 Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 13:37:32 +0200 Subject: [PATCH 05/36] Fix: Mypy error --- src/aleph/web/controllers/storage.py | 65 +++++++++++++++------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index e265889d5..9e4c479cf 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -1,49 +1,36 @@ -import asyncio import base64 import datetime as dt import functools import logging from hashlib import sha256 -from io import StringIO -from typing import Union, Tuple, Dict, Optional +from typing import Union, Tuple import aio_pika from eth_account import Account from eth_account.messages import encode_defunct from aleph.chains.common import get_verification_buffer -from aleph.jobs.process_pending_messages import PendingMessageProcessor from aiohttp import web from aiohttp.web_request import FileField from aleph_message.models import ItemType from multidict import MultiDictProxy - -from aleph.chains.chain_service import ChainService, LOGGER -from aleph.chains.nuls import NulsConnector from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.files import count_file_pins, get_file -from aleph.db.accessors.messages import get_message_status, message_exists -from aleph.db.connection import make_session_factory from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError -from aleph.services.p2p import init_p2p_client -from aleph.services.storage.engine import StorageEngine -from aleph.services.storage.fileystem_engine import FileSystemStorageEngine -from aleph.storage import StorageService from aleph.toolkit import json -from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, - get_storage_service_from_request, get_mq_channel_from_request, get_config_from_request, get_mq_conn_from_request, + get_storage_service_from_request, + get_config_from_request, ) from aleph.web.controllers.utils import multidict_proxy_to_io from aleph.schemas.pending_messages import BasePendingMessage logger = logging.getLogger(__name__) -from aleph.schemas.pending_messages import parse_message MAX_FILE_SIZE = 100 * 1024 * 1024 @@ -111,34 +98,47 @@ async def verify_signature(message: BasePendingMessage) -> bool: return verified -async def get_message_content(post_data: MultiDictProxy[Union[str, bytes, FileField]]) -> Tuple[dict, int]: +async def get_message_content( + post_data: MultiDictProxy[Union[str, bytes, FileField]] +) -> Tuple[dict, int]: message_bytearray = post_data.get("message", b"") value = post_data.get("size") or 0 - if not message_bytearray: - return {}, int(value) # Empty dictionary if no message content - message_string = message_bytearray.decode("utf-8") - message_dict = json.loads(message_string) - message_dict["time"] = float(message_dict["time"]) + if isinstance(message_bytearray, bytearray): + message_string = message_bytearray.decode("utf-8") + message_dict = json.loads(message_string) + message_dict["time"] = float(message_dict["time"]) + else: + message_dict = {} - return message_dict, int(value) + return message_dict, int(str(value)) async def init_mq_con(config): return await aio_pika.connect_robust( - host=config.p2p.mq_host.value, port=config.rabbitmq.port.value, login=config.rabbitmq.username.value, - password=config.rabbitmq.password.value + host=config.p2p.mq_host.value, + port=config.rabbitmq.port.value, + login=config.rabbitmq.username.value, + password=config.rabbitmq.password.value, ) -async def verify_and_handle_request(pending_message_db, file_io, message, size): +async def verify_and_handle_request( + pending_message_db, file_io, message, size, session_factory +): content = file_io.read(size) item_content = json.loads(message["item_content"]) actual_item_hash = sha256(content).hexdigest() c_item_hash = item_content["item_hash"] is_signature = await verify_signature(message=pending_message_db) - + with session_factory() as session: + current_balance = get_total_balance( + session=session, address=pending_message_db.sender + ) + if current_balance < len(content): + output = {"status": "Payment Required"} + return web.json_response(output, status=402) if not is_signature: output = {"status": "Forbidden"} return web.json_response(output, status=403) @@ -161,9 +161,12 @@ async def storage_add_file_with_message(request: web.Request): post = await request.post() file_io = multidict_proxy_to_io(post) message, size = await get_message_content(post) - pending_message_db = PendingMessageDb.from_message_dict(message_dict=message, reception_time=dt.datetime.now(), - fetched=True) - is_valid_message = await verify_and_handle_request(pending_message_db, file_io, message, size) + pending_message_db = PendingMessageDb.from_message_dict( + message_dict=message, reception_time=dt.datetime.now(), fetched=True + ) + is_valid_message = await verify_and_handle_request( + pending_message_db, file_io, message, size, session_factory + ) if is_valid_message is not None: return is_valid_message From 35a2c163451fd4dda60463b179b6af5dbfd99a57 Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 16:18:40 +0200 Subject: [PATCH 06/36] Refactor: Use MQ queue to trace status of the message --- src/aleph/web/controllers/storage.py | 31 +++++++-- tests/api/test_storage.py | 97 ++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 9e4c479cf..0285fc604 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -21,13 +21,22 @@ from aleph.exceptions import AlephStorageException, UnknownHashError from aleph.toolkit import json from aleph.types.db_session import DbSession +from aleph.types.message_status import MessageProcessingStatus from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, get_storage_service_from_request, get_config_from_request, + get_mq_channel_from_request, +) +from aleph.web.controllers.p2p import ( + _mq_read_one_message, + _processing_status_to_http_status, +) +from aleph.web.controllers.utils import ( + multidict_proxy_to_io, + mq_make_aleph_message_topic_queue, ) -from aleph.web.controllers.utils import multidict_proxy_to_io from aleph.schemas.pending_messages import BasePendingMessage logger = logging.getLogger(__name__) @@ -156,7 +165,6 @@ async def storage_add_file_with_message(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) config = get_config_from_request(request) - mq_con = init_mq_con(config) post = await request.post() file_io = multidict_proxy_to_io(post) @@ -164,6 +172,13 @@ async def storage_add_file_with_message(request: web.Request): pending_message_db = PendingMessageDb.from_message_dict( message_dict=message, reception_time=dt.datetime.now(), fetched=True ) + mq_channel = await get_mq_channel_from_request(request, logger=logger) + mq_queue = await mq_make_aleph_message_topic_queue( + channel=mq_channel, + config=config, + routing_key=f"*.{pending_message_db.item_hash}", + ) + is_valid_message = await verify_and_handle_request( pending_message_db, file_io, message, size, session_factory ) @@ -176,8 +191,16 @@ async def storage_add_file_with_message(request: web.Request): ) session.add(pending_message_db) session.commit() - output = {"status": "success", "hash": file_hash} - return web.json_response(output) + mq_message = await _mq_read_one_message(mq_queue, 30) + + if mq_message is None: + output = {"status": "accepted"} + return web.json_response(output, status=202) + if mq_message.routing_key is not None: + status_str, _item_hash = mq_message.routing_key.split(".") + processing_status = MessageProcessingStatus(status_str) + status_code = _processing_status_to_http_status(processing_status) + return web.json_response(status=status_code, text=file_hash) async def storage_add_file(request: web.Request): diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 767f9ea97..9a6e58fcc 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -27,6 +27,9 @@ from in_memory_storage_engine import InMemoryStorageEngine import datetime as dt + +from aleph.web.controllers.app_state_getters import get_mq_channel_from_request + IPFS_ADD_FILE_URI = "/api/v0/ipfs/add_file" IPFS_ADD_JSON_URI = "/api/v0/ipfs/add_json" STORAGE_ADD_FILE_URI = "/api/v0/storage/add_file" @@ -138,7 +141,64 @@ async def add_file_with_message( size: str, error_code: int, balance: int, + mocker, ): + mocker.patch("aleph.web.controllers.storage.get_mq_channel_from_request") + mocked_queue = mocker.patch( + "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" + ) + + # Create a mock MQ response object + mock_mq_message = mocker.Mock() + mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" + mocker.patch( + "aleph.web.controllers.storage._mq_read_one_message", + return_value=mock_mq_message, + ) + + with session_factory() as session: + session.add( + AlephBalanceDb( + address="0x6dA130FD646f826C1b8080C07448923DF9a79aaA", + chain=Chain.ETH, + balance=Decimal(balance), + eth_height=0, + ) + ) + session.commit() + + json_data = json.dumps(MESSAGE_DICT) + form_data = aiohttp.FormData() + + form_data.add_field("file", file_content) + form_data.add_field("message", json_data, content_type="application/json") + form_data.add_field("size", size) + response = await api_client.post(uri, data=form_data) + assert response.status == error_code, await response.text() + + +async def add_file_with_message_202( + api_client, + session_factory: DbSessionFactory, + uri: str, + file_content: bytes, + size: str, + error_code: int, + balance: int, + mocker, +): + mocker.patch("aleph.web.controllers.storage.get_mq_channel_from_request") + mocked_queue = mocker.patch( + "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" + ) + + # Create a mock MQ response object + mock_mq_message = mocker.Mock() + mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" + mocker.patch( + "aleph.web.controllers.storage._mq_read_one_message", return_value=None + ) + with session_factory() as session: session.add( AlephBalanceDb( @@ -206,6 +266,7 @@ async def test_storage_add_file_with_message( size, error_code, balance, + mocker, ): await add_file_with_message( api_client, @@ -215,6 +276,42 @@ async def test_storage_add_file_with_message( size=size, error_code=int(error_code), balance=int(balance), + mocker=mocker, + ) + + +@pytest.mark.parametrize( + "file_content, expected_hash, size, error_code, balance", + [ + ( + b"Hello Aleph.im\n", + "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + "15", + "202", + "1000", + ), + ], +) +@pytest.mark.asyncio +async def test_storage_add_file_with_message_202( + api_client, + session_factory: DbSessionFactory, + file_content, + expected_hash, + size, + error_code, + balance, + mocker, +): + await add_file_with_message_202( + api_client, + session_factory, + uri=STORAGE_ADD_FILE_URI, + file_content=file_content, + size=size, + error_code=int(error_code), + balance=int(balance), + mocker=mocker, ) From ce83b881902102a0b4d226ea60a02204e929a2cc Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 16:23:04 +0200 Subject: [PATCH 07/36] Fix: Test --- tests/api/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 9a6e58fcc..0b2486cde 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -110,7 +110,7 @@ async def add_file( form_data = aiohttp.FormData() form_data.add_field("file", file_content) - post_response = await api_client.post(uri, data=form_data, sync=True) + post_response = await api_client.post(uri, data=form_data) assert post_response.status == 200, await post_response.text() post_response_json = await post_response.json() assert post_response_json["status"] == "success" From 41e1a86fd54bf10a614f0e4bde18f53b069e4c93 Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 16:29:46 +0200 Subject: [PATCH 08/36] Fix: the message was not commit to the db --- src/aleph/web/controllers/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 0285fc604..a58404fbb 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -215,6 +215,7 @@ async def storage_add_file(request: web.Request): file_hash = await storage_service.add_file( session=session, fileobject=file_io, engine=ItemType.storage ) + session.commit() output = {"status": "success", "hash": file_hash} return web.json_response(output) From 23ecf40fd5bc014527e561f8b000fed620e586cc Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 22 Aug 2023 16:35:36 +0200 Subject: [PATCH 09/36] Fix : Remove unused function --- src/aleph/web/controllers/storage.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index a58404fbb..8eb6cb830 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -123,15 +123,6 @@ async def get_message_content( return message_dict, int(str(value)) -async def init_mq_con(config): - return await aio_pika.connect_robust( - host=config.p2p.mq_host.value, - port=config.rabbitmq.port.value, - login=config.rabbitmq.username.value, - password=config.rabbitmq.password.value, - ) - - async def verify_and_handle_request( pending_message_db, file_io, message, size, session_factory ): From b45ecaaab3e02920b45dd9a85f777b27c658293f Mon Sep 17 00:00:00 2001 From: 1yam Date: Wed, 23 Aug 2023 12:26:33 +0200 Subject: [PATCH 10/36] Refactor: Move private functions to public --- src/aleph/web/controllers/p2p.py | 39 +++------------------------- src/aleph/web/controllers/storage.py | 11 +++----- src/aleph/web/controllers/utils.py | 35 +++++++++++++++++++++++++ tests/api/test_storage.py | 4 +-- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index 6a33bb5ba..e7d9c1dc4 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -26,7 +26,8 @@ get_p2p_client_from_request, get_mq_channel_from_request, ) -from aleph.web.controllers.utils import mq_make_aleph_message_topic_queue +from aleph.web.controllers.utils import mq_make_aleph_message_topic_queue, processing_status_to_http_status, \ + mq_read_one_message LOGGER = logging.getLogger(__name__) @@ -142,38 +143,6 @@ async def pub_json(request: web.Request): ) -async def _mq_read_one_message( - mq_queue: aio_pika.abc.AbstractQueue, timeout: float -) -> Optional[aio_pika.abc.AbstractIncomingMessage]: - """ - Consume one element from a message queue and then return. - """ - - queue: asyncio.Queue = asyncio.Queue() - - async def _process_message(message: aio_pika.abc.AbstractMessage): - await queue.put(message) - - consumer_tag = await mq_queue.consume(_process_message, no_ack=True) - - try: - return await asyncio.wait_for(queue.get(), timeout) - except asyncio.TimeoutError: - return None - finally: - await mq_queue.cancel(consumer_tag) - - -def _processing_status_to_http_status(status: MessageProcessingStatus) -> int: - mapping = { - MessageProcessingStatus.PROCESSED_NEW_MESSAGE: 200, - MessageProcessingStatus.PROCESSED_CONFIRMATION: 200, - MessageProcessingStatus.FAILED_WILL_RETRY: 202, - MessageProcessingStatus.FAILED_REJECTED: 422, - } - return mapping[status] - - class PubMessageRequest(BaseModel): sync: bool = False message_dict: Dict[str, Any] = Field(alias="message") @@ -249,7 +218,7 @@ async def pub_message(request: web.Request): # Ignore type checking here, we know that mq_queue is set at this point assert mq_queue is not None - response = await _mq_read_one_message(mq_queue, timeout=30) + response = await mq_read_one_message(mq_queue, timeout=30) # Delete the queue immediately await mq_queue.delete(if_empty=False) @@ -262,7 +231,7 @@ async def pub_message(request: web.Request): assert routing_key is not None # again, for type checking status_str, _item_hash = routing_key.split(".") processing_status = MessageProcessingStatus(status_str) - status_code = _processing_status_to_http_status(processing_status) + status_code = processing_status_to_http_status(processing_status) status.message_status = processing_status.to_message_status() diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 8eb6cb830..0557ee5b9 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -29,13 +29,10 @@ get_config_from_request, get_mq_channel_from_request, ) -from aleph.web.controllers.p2p import ( - _mq_read_one_message, - _processing_status_to_http_status, -) + from aleph.web.controllers.utils import ( multidict_proxy_to_io, - mq_make_aleph_message_topic_queue, + mq_make_aleph_message_topic_queue, processing_status_to_http_status, mq_read_one_message, ) from aleph.schemas.pending_messages import BasePendingMessage @@ -182,7 +179,7 @@ async def storage_add_file_with_message(request: web.Request): ) session.add(pending_message_db) session.commit() - mq_message = await _mq_read_one_message(mq_queue, 30) + mq_message = await mq_read_one_message(mq_queue, 30) if mq_message is None: output = {"status": "accepted"} @@ -190,7 +187,7 @@ async def storage_add_file_with_message(request: web.Request): if mq_message.routing_key is not None: status_str, _item_hash = mq_message.routing_key.split(".") processing_status = MessageProcessingStatus(status_str) - status_code = _processing_status_to_http_status(processing_status) + status_code = processing_status_to_http_status(processing_status) return web.json_response(status=status_code, text=file_hash) diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index b8267cb3b..1d638e1cf 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -1,3 +1,4 @@ +import asyncio import json from io import BytesIO, StringIO from math import ceil @@ -10,6 +11,8 @@ from configmanager import Config from multidict import MultiDictProxy +from aleph.types.message_status import MessageProcessingStatus + DEFAULT_MESSAGES_PER_PAGE = 20 DEFAULT_PAGE = 1 LIST_FIELD_SEPARATOR = "," @@ -160,3 +163,35 @@ async def mq_make_aleph_message_topic_queue( ) await mq_queue.bind(mq_message_exchange, routing_key=routing_key) return mq_queue + + +def processing_status_to_http_status(status: MessageProcessingStatus) -> int: + mapping = { + MessageProcessingStatus.PROCESSED_NEW_MESSAGE: 200, + MessageProcessingStatus.PROCESSED_CONFIRMATION: 200, + MessageProcessingStatus.FAILED_WILL_RETRY: 202, + MessageProcessingStatus.FAILED_REJECTED: 422, + } + return mapping[status] + + +async def mq_read_one_message( + mq_queue: aio_pika.abc.AbstractQueue, timeout: float +) -> Optional[aio_pika.abc.AbstractIncomingMessage]: + """ + Consume one element from a message queue and then return. + """ + + queue: asyncio.Queue = asyncio.Queue() + + async def _process_message(message: aio_pika.abc.AbstractMessage): + await queue.put(message) + + consumer_tag = await mq_queue.consume(_process_message, no_ack=True) + + try: + return await asyncio.wait_for(queue.get(), timeout) + except asyncio.TimeoutError: + return None + finally: + await mq_queue.cancel(consumer_tag) \ No newline at end of file diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 0b2486cde..84628dcca 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -152,7 +152,7 @@ async def add_file_with_message( mock_mq_message = mocker.Mock() mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" mocker.patch( - "aleph.web.controllers.storage._mq_read_one_message", + "aleph.web.controllers.storage.mq_read_one_message", return_value=mock_mq_message, ) @@ -196,7 +196,7 @@ async def add_file_with_message_202( mock_mq_message = mocker.Mock() mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" mocker.patch( - "aleph.web.controllers.storage._mq_read_one_message", return_value=None + "aleph.web.controllers.storage.mq_read_one_message", return_value=None ) with session_factory() as session: From fdb8af91ef5ce540d952d3667fc09b76193e835e Mon Sep 17 00:00:00 2001 From: 1yam Date: Thu, 24 Aug 2023 12:44:14 +0200 Subject: [PATCH 11/36] Fix: use ChainService instead of re create verify_signature --- src/aleph/api_entrypoint.py | 10 ++- .../web/controllers/app_state_getters.py | 7 ++- src/aleph/web/controllers/storage.py | 63 +++++++++---------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/src/aleph/api_entrypoint.py b/src/aleph/api_entrypoint.py index 1f5d8b506..84bd592de 100644 --- a/src/aleph/api_entrypoint.py +++ b/src/aleph/api_entrypoint.py @@ -6,6 +6,7 @@ from configmanager import Config import aleph.config +from aleph.chains.chain_service import ChainService from aleph.db.connection import make_engine, make_session_factory from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService @@ -21,7 +22,10 @@ APP_STATE_NODE_CACHE, APP_STATE_P2P_CLIENT, APP_STATE_SESSION_FACTORY, - APP_STATE_STORAGE_SERVICE, APP_STATE_MQ_CHANNEL, APP_STATE_MQ_WS_CHANNEL, + APP_STATE_STORAGE_SERVICE, + APP_STATE_MQ_CHANNEL, + APP_STATE_MQ_WS_CHANNEL, + APP_STATE_CHAIN_SERVICE, ) @@ -49,6 +53,9 @@ async def configure_aiohttp_app( ipfs_service=ipfs_service, node_cache=node_cache, ) + chain_service = ChainService( + storage_service=storage_service, session_factory=session_factory + ) app = create_aiohttp_app() @@ -67,6 +74,7 @@ async def configure_aiohttp_app( app[APP_STATE_NODE_CACHE] = node_cache app[APP_STATE_STORAGE_SERVICE] = storage_service app[APP_STATE_SESSION_FACTORY] = session_factory + # app[APP_STATE_CHAIN_SERVICE] = chain_service return app diff --git a/src/aleph/web/controllers/app_state_getters.py b/src/aleph/web/controllers/app_state_getters.py index d6a25b9bf..0b4b3d65b 100644 --- a/src/aleph/web/controllers/app_state_getters.py +++ b/src/aleph/web/controllers/app_state_getters.py @@ -11,6 +11,7 @@ from aleph_p2p_client import AlephP2PServiceClient from configmanager import Config +from aleph.chains.chain_service import ChainService from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService from aleph.storage import StorageService @@ -27,7 +28,7 @@ APP_STATE_P2P_CLIENT = "p2p_client" APP_STATE_SESSION_FACTORY = "session_factory" APP_STATE_STORAGE_SERVICE = "storage_service" - +APP_STATE_CHAIN_SERVICE = "chain_service" T = TypeVar("T") @@ -103,3 +104,7 @@ def get_session_factory_from_request(request: web.Request) -> DbSessionFactory: def get_storage_service_from_request(request: web.Request) -> StorageService: return cast(StorageService, request.app[APP_STATE_STORAGE_SERVICE]) + + +def get_chain_service_from_request(request: web.Request) -> ChainService: + return cast(ChainService, request.app[APP_STATE_CHAIN_SERVICE]) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 0557ee5b9..69849746a 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -9,6 +9,7 @@ from eth_account import Account from eth_account.messages import encode_defunct +from aleph.chains.chain_service import ChainService from aleph.chains.common import get_verification_buffer from aiohttp import web @@ -21,18 +22,21 @@ from aleph.exceptions import AlephStorageException, UnknownHashError from aleph.toolkit import json from aleph.types.db_session import DbSession -from aleph.types.message_status import MessageProcessingStatus +from aleph.types.message_status import MessageProcessingStatus, InvalidSignature from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, get_storage_service_from_request, get_config_from_request, get_mq_channel_from_request, + get_chain_service_from_request, ) from aleph.web.controllers.utils import ( multidict_proxy_to_io, - mq_make_aleph_message_topic_queue, processing_status_to_http_status, mq_read_one_message, + mq_make_aleph_message_topic_queue, + processing_status_to_http_status, + mq_read_one_message, ) from aleph.schemas.pending_messages import BasePendingMessage @@ -77,33 +81,6 @@ async def add_storage_json_controller(request: web.Request): return web.json_response(output) -async def verify_signature(message: BasePendingMessage) -> bool: - """Verifies a signature of a message, return True if verified, false if not""" - verification = get_verification_buffer(message) - - message_hash = await run_in_executor( - None, functools.partial(encode_defunct, text=verification.decode("utf-8")) - ) - - verified = False - try: - # we assume the signature is a valid string - address = await run_in_executor( - None, - functools.partial( - Account.recover_message, message_hash, signature=message.signature - ), - ) - if address == message.sender: - verified = True - else: - return False - - except Exception as e: - verified = False - return verified - - async def get_message_content( post_data: MultiDictProxy[Union[str, bytes, FileField]] ) -> Tuple[dict, int]: @@ -121,14 +98,24 @@ async def get_message_content( async def verify_and_handle_request( - pending_message_db, file_io, message, size, session_factory + pending_message_db, + file_io, + message, + size, + session_factory, + chain_service: ChainService, ): content = file_io.read(size) item_content = json.loads(message["item_content"]) actual_item_hash = sha256(content).hexdigest() c_item_hash = item_content["item_hash"] - is_signature = await verify_signature(message=pending_message_db) + try: + await chain_service.verify_signature(pending_message_db) + except InvalidSignature: + output = {"status": "Forbidden"} + return web.json_response(output, status=403) + with session_factory() as session: current_balance = get_total_balance( session=session, address=pending_message_db.sender @@ -136,9 +123,6 @@ async def verify_and_handle_request( if current_balance < len(content): output = {"status": "Payment Required"} return web.json_response(output, status=402) - if not is_signature: - output = {"status": "Forbidden"} - return web.json_response(output, status=403) elif actual_item_hash != c_item_hash: output = {"status": "Unprocessable Content"} return web.json_response(output, status=422) @@ -152,6 +136,10 @@ async def verify_and_handle_request( async def storage_add_file_with_message(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) + # TODO : Add chainservice to ccn_api_client to be able to call get_chainservice_from_request + chain_service: ChainService = ChainService( + session_factory=session_factory, storage_service=storage_service + ) config = get_config_from_request(request) post = await request.post() @@ -168,7 +156,12 @@ async def storage_add_file_with_message(request: web.Request): ) is_valid_message = await verify_and_handle_request( - pending_message_db, file_io, message, size, session_factory + pending_message_db, + file_io, + message, + size, + session_factory, + chain_service, ) if is_valid_message is not None: return is_valid_message From 1d1efb44416e79a9d335c06aeed9fb9128df75c3 Mon Sep 17 00:00:00 2001 From: 1yam Date: Thu, 24 Aug 2023 12:47:41 +0200 Subject: [PATCH 12/36] Fix: missed to push it --- tests/api/test_storage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 84628dcca..f13a3b836 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -10,6 +10,7 @@ from aleph_message.models import ItemHash, MessageType, Chain, ItemType from configmanager import Config +from aleph.chains.chain_service import ChainService from aleph.handlers.message_handler import MessageHandler from aleph.schemas.pending_messages import ( parse_message, @@ -147,6 +148,7 @@ async def add_file_with_message( mocked_queue = mocker.patch( "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" ) + mocker.patch("aleph.web.controllers.storage.get_chain_service_from_request") # Create a mock MQ response object mock_mq_message = mocker.Mock() @@ -191,13 +193,12 @@ async def add_file_with_message_202( mocked_queue = mocker.patch( "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" ) + mocker.patch("aleph.web.controllers.storage.get_chain_service_from_request") # Create a mock MQ response object mock_mq_message = mocker.Mock() mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" - mocker.patch( - "aleph.web.controllers.storage.mq_read_one_message", return_value=None - ) + mocker.patch("aleph.web.controllers.storage.mq_read_one_message", return_value=None) with session_factory() as session: session.add( From 55e22831314a26e6b78a9b0091000b28801f9d5b Mon Sep 17 00:00:00 2001 From: 1yam Date: Thu, 24 Aug 2023 12:48:33 +0200 Subject: [PATCH 13/36] Fix: Use MiB instead of MB --- src/aleph/web/controllers/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 69849746a..7a722f94f 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -8,6 +8,7 @@ import aio_pika from eth_account import Account from eth_account.messages import encode_defunct +from mypy.dmypy_server import MiB from aleph.chains.chain_service import ChainService from aleph.chains.common import get_verification_buffer @@ -126,7 +127,7 @@ async def verify_and_handle_request( elif actual_item_hash != c_item_hash: output = {"status": "Unprocessable Content"} return web.json_response(output, status=422) - elif len(content) > 25_000 and not message: + elif len(content) > 25 * MiB and not message: output = {"status": "Unauthorized"} return web.json_response(output, status=401) else: From 45809a1ea4cddd984480d5810aeec80d5e3c776c Mon Sep 17 00:00:00 2001 From: 1yam Date: Thu, 24 Aug 2023 12:55:42 +0200 Subject: [PATCH 14/36] Fix --- src/aleph/web/controllers/storage.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 7a722f94f..dd8c11924 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -86,7 +86,7 @@ async def get_message_content( post_data: MultiDictProxy[Union[str, bytes, FileField]] ) -> Tuple[dict, int]: message_bytearray = post_data.get("message", b"") - value = post_data.get("size") or 0 + file_size = post_data.get("file_size") or 0 if isinstance(message_bytearray, bytearray): message_string = message_bytearray.decode("utf-8") @@ -95,7 +95,7 @@ async def get_message_content( else: message_dict = {} - return message_dict, int(str(value)) + return message_dict, int(str(file_size)) async def verify_and_handle_request( @@ -124,9 +124,13 @@ async def verify_and_handle_request( if current_balance < len(content): output = {"status": "Payment Required"} return web.json_response(output, status=402) + elif len(content) > (1000 * MiB): + output = {"status": "Payload too large"} + return web.json_response(output, status=413) elif actual_item_hash != c_item_hash: output = {"status": "Unprocessable Content"} return web.json_response(output, status=422) + elif len(content) > 25 * MiB and not message: output = {"status": "Unauthorized"} return web.json_response(output, status=401) @@ -187,7 +191,7 @@ async def storage_add_file_with_message(request: web.Request): async def storage_add_file(request: web.Request): post = await request.post() - if post.get("message", b"") is not None and post.get("size") is not None: + if post.get("message", b"") is not None and post.get("file_size") is not None: return await storage_add_file_with_message(request) storage_service = get_storage_service_from_request(request) From 455a4b843041cd0441cfa950462fb1f12d37d6b9 Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 12:57:10 +0200 Subject: [PATCH 15/36] Refactor: storage_add --- src/aleph/web/controllers/p2p.py | 13 +-- src/aleph/web/controllers/storage.py | 165 +++++++++++++-------------- src/aleph/web/controllers/utils.py | 14 ++- tests/api/test_storage.py | 6 +- 4 files changed, 100 insertions(+), 98 deletions(-) diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index e7d9c1dc4..118a0e8f1 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -27,7 +27,7 @@ get_mq_channel_from_request, ) from aleph.web.controllers.utils import mq_make_aleph_message_topic_queue, processing_status_to_http_status, \ - mq_read_one_message + mq_read_one_message, validate_message_dict LOGGER = logging.getLogger(__name__) @@ -46,13 +46,6 @@ def from_failures(cls, failed_publications: List[Protocol]): return cls(status=status, failed=failed_publications) -def _validate_message_dict(message_dict: Mapping[str, Any]) -> BasePendingMessage: - try: - return parse_message(message_dict) - except InvalidMessageException as e: - raise web.HTTPUnprocessableEntity(body=str(e)) - - def _validate_request_data(config: Config, request_data: Dict) -> None: """ Validates the content of a JSON pubsub message depending on the channel @@ -84,7 +77,7 @@ def _validate_request_data(config: Config, request_data: Dict) -> None: reason="'data': must be deserializable as JSON." ) - _validate_message_dict(message_dict) + validate_message_dict(message_dict) async def _pub_on_p2p_topics( @@ -163,7 +156,7 @@ async def pub_message(request: web.Request): # Body must be valid JSON raise web.HTTPUnprocessableEntity() - pending_message = _validate_message_dict(request_data.message_dict) + pending_message = validate_message_dict(request_data.message_dict) # In sync mode, wait for a message processing event. We need to create the queue # before publishing the message on P2P topics in order to guarantee that the event diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index dd8c11924..49037ef61 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -1,3 +1,4 @@ +import ast import base64 import datetime as dt import functools @@ -6,9 +7,11 @@ from typing import Union, Tuple import aio_pika +from aiohttp.web_response import Response from eth_account import Account from eth_account.messages import encode_defunct from mypy.dmypy_server import MiB +from sqlalchemy.orm import Session from aleph.chains.chain_service import ChainService from aleph.chains.common import get_verification_buffer @@ -22,8 +25,8 @@ from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError from aleph.toolkit import json -from aleph.types.db_session import DbSession -from aleph.types.message_status import MessageProcessingStatus, InvalidSignature +from aleph.types.db_session import DbSession, DbSessionFactory +from aleph.types.message_status import MessageProcessingStatus, InvalidSignature, InvalidMessageException from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, @@ -37,9 +40,9 @@ multidict_proxy_to_io, mq_make_aleph_message_topic_queue, processing_status_to_http_status, - mq_read_one_message, + mq_read_one_message, validate_message_dict, ) -from aleph.schemas.pending_messages import BasePendingMessage +from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage logger = logging.getLogger(__name__) @@ -83,7 +86,7 @@ async def add_storage_json_controller(request: web.Request): async def get_message_content( - post_data: MultiDictProxy[Union[str, bytes, FileField]] + post_data: MultiDictProxy[Union[str, bytes, FileField]] ) -> Tuple[dict, int]: message_bytearray = post_data.get("message", b"") file_size = post_data.get("file_size") or 0 @@ -98,109 +101,105 @@ async def get_message_content( return message_dict, int(str(file_size)) -async def verify_and_handle_request( - pending_message_db, - file_io, - message, - size, - session_factory, - chain_service: ChainService, -): +async def _verify_message_signature(pending_message_db: PendingMessageDb, chain_service: ChainService) -> None: + try: + await chain_service.verify_signature(pending_message_db) + except InvalidSignature: + raise web.HTTPForbidden() + + +async def _verify_user_balance(pending_message_db: PendingMessageDb, session: DbSession, size: int) -> None: + current_balance = get_total_balance( + session=session, address=pending_message_db.sender + ) + required_balance = (size / MiB) / 3 + # Need to merge to get this functions + # current_cost_for_user = get_total_cost_for_address( + # session=session, address=pending_message_db.sender + # ) + if current_balance < required_balance: + raise web.HTTPPaymentRequired + +async def _verify_user_file(message : dict, size : int, file_io) -> None: + file_io.seek(0) content = file_io.read(size) item_content = json.loads(message["item_content"]) actual_item_hash = sha256(content).hexdigest() - c_item_hash = item_content["item_hash"] + client_item_hash = item_content["item_hash"] + if len(content) > (1000 * MiB): + raise web.HTTPRequestEntityTooLarge(actual_size=len(content), max_size=(1000 * MiB)) + elif actual_item_hash != client_item_hash: + raise web.HTTPUnprocessableEntity() + + +async def storage_add_file_with_message(request: web.Request, session: DbSession, chain_service, post, file_io, sync = False): + config = get_config_from_request(request) + message, size = await get_message_content(post) + mq_queue = None try: - await chain_service.verify_signature(pending_message_db) - except InvalidSignature: - output = {"status": "Forbidden"} - return web.json_response(output, status=403) + valid_message: BasePendingMessage = validate_message_dict(message) + except InvalidMessageException: + output = {"status": "rejected"} + return web.json_response(output, status=422) - with session_factory() as session: - current_balance = get_total_balance( - session=session, address=pending_message_db.sender + pending_store_message = PendingStoreMessage.parse_obj(valid_message) + pending_message_db = PendingMessageDb.from_obj( + obj=pending_store_message, reception_time=dt.datetime.now(), fetched=True + ) + await _verify_message_signature( + pending_message_db=pending_message_db, chain_service=chain_service + ) + await _verify_user_balance(session=session, pending_message_db=pending_message_db, size=size) + await _verify_user_file(message=message, size=size, file_io=file_io) + + if sync: + mq_channel = await get_mq_channel_from_request(request, logger=logger) + mq_queue = await mq_make_aleph_message_topic_queue( + channel=mq_channel, + config=config, + routing_key=f"*.{pending_message_db.item_hash}", ) - if current_balance < len(content): - output = {"status": "Payment Required"} - return web.json_response(output, status=402) - elif len(content) > (1000 * MiB): - output = {"status": "Payload too large"} - return web.json_response(output, status=413) - elif actual_item_hash != c_item_hash: - output = {"status": "Unprocessable Content"} - return web.json_response(output, status=422) - elif len(content) > 25 * MiB and not message: - output = {"status": "Unauthorized"} - return web.json_response(output, status=401) - else: - return None + session.add(pending_message_db) + session.commit() + if sync: + mq_message = await mq_read_one_message(mq_queue, 30) + if mq_message is None: + raise web.HTTPAccepted() + if mq_message.routing_key is not None: + status_str, _item_hash = mq_message.routing_key.split(".") + processing_status = MessageProcessingStatus(status_str) + status_code = processing_status_to_http_status(processing_status) + return web.json_response(status=status_code, text=_item_hash) -async def storage_add_file_with_message(request: web.Request): + +async def storage_add_file(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) # TODO : Add chainservice to ccn_api_client to be able to call get_chainservice_from_request chain_service: ChainService = ChainService( session_factory=session_factory, storage_service=storage_service ) - config = get_config_from_request(request) - post = await request.post() file_io = multidict_proxy_to_io(post) - message, size = await get_message_content(post) - pending_message_db = PendingMessageDb.from_message_dict( - message_dict=message, reception_time=dt.datetime.now(), fetched=True - ) - mq_channel = await get_mq_channel_from_request(request, logger=logger) - mq_queue = await mq_make_aleph_message_topic_queue( - channel=mq_channel, - config=config, - routing_key=f"*.{pending_message_db.item_hash}", - ) - - is_valid_message = await verify_and_handle_request( - pending_message_db, - file_io, - message, - size, - session_factory, - chain_service, - ) - if is_valid_message is not None: - return is_valid_message - - with session_factory() as session: - file_hash = await storage_service.add_file( - session=session, fileobject=file_io, engine=ItemType.storage - ) - session.add(pending_message_db) - session.commit() - mq_message = await mq_read_one_message(mq_queue, 30) - - if mq_message is None: - output = {"status": "accepted"} - return web.json_response(output, status=202) - if mq_message.routing_key is not None: - status_str, _item_hash = mq_message.routing_key.split(".") - processing_status = MessageProcessingStatus(status_str) - status_code = processing_status_to_http_status(processing_status) - return web.json_response(status=status_code, text=file_hash) + post = await request.post() + sync = False + if post.get("message", b"") is None: + raise web.HTTPUnauthorized() -async def storage_add_file(request: web.Request): - post = await request.post() - if post.get("message", b"") is not None and post.get("file_size") is not None: - return await storage_add_file_with_message(request) + sync_value = post.get("sync") + if sync_value is not None: + sync = ast.literal_eval(sync_value) - storage_service = get_storage_service_from_request(request) - session_factory = get_session_factory_from_request(request) - file_io = multidict_proxy_to_io(post) with session_factory() as session: file_hash = await storage_service.add_file( session=session, fileobject=file_io, engine=ItemType.storage ) + if post.get("message", b"") is not None and post.get("file_size") is not None: + await storage_add_file_with_message(request, session, chain_service, post, file_io, sync) session.commit() output = {"status": "success", "hash": file_hash} return web.json_response(output) diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index 1d638e1cf..b03744f49 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -2,7 +2,7 @@ import json from io import BytesIO, StringIO from math import ceil -from typing import Optional, Union, IO +from typing import Optional, Union, IO, Mapping, Any import aio_pika import aiohttp_jinja2 @@ -11,7 +11,8 @@ from configmanager import Config from multidict import MultiDictProxy -from aleph.types.message_status import MessageProcessingStatus +from aleph.schemas.pending_messages import BasePendingMessage, parse_message +from aleph.types.message_status import MessageProcessingStatus, InvalidMessageException DEFAULT_MESSAGES_PER_PAGE = 20 DEFAULT_PAGE = 1 @@ -194,4 +195,11 @@ async def _process_message(message: aio_pika.abc.AbstractMessage): except asyncio.TimeoutError: return None finally: - await mq_queue.cancel(consumer_tag) \ No newline at end of file + await mq_queue.cancel(consumer_tag) + + +def validate_message_dict(message_dict: Mapping[str, Any]) -> BasePendingMessage: + try: + return parse_message(message_dict) + except InvalidMessageException as e: + raise web.HTTPUnprocessableEntity(body=str(e)) \ No newline at end of file diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index f13a3b836..425e7d1c9 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -174,7 +174,8 @@ async def add_file_with_message( form_data.add_field("file", file_content) form_data.add_field("message", json_data, content_type="application/json") - form_data.add_field("size", size) + form_data.add_field("file_size", size) + response = await api_client.post(uri, data=form_data) assert response.status == error_code, await response.text() @@ -216,7 +217,8 @@ async def add_file_with_message_202( form_data.add_field("file", file_content) form_data.add_field("message", json_data, content_type="application/json") - form_data.add_field("size", size) + form_data.add_field("file_size", size) + form_data.add_field("sync", "True") response = await api_client.post(uri, data=form_data) assert response.status == error_code, await response.text() From 11451e1cd100c2dcd77dcceaf1cd2a75a4dc3260 Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 13:03:54 +0200 Subject: [PATCH 16/36] Fix: Miss one conditions --- src/aleph/web/controllers/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 49037ef61..e0bad11a9 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -187,8 +187,9 @@ async def storage_add_file(request: web.Request): post = await request.post() sync = False - if post.get("message", b"") is None: + if post.get("message", b"") is None and len(file_io.read()) > (25 * MiB): raise web.HTTPUnauthorized() + file_io.seek(0) sync_value = post.get("sync") if sync_value is not None: From 85745cc6e0b4f30d45ef1c0d09879045c6710b99 Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 13:33:22 +0200 Subject: [PATCH 17/36] Fix: mypy error --- src/aleph/web/controllers/storage.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index e0bad11a9..c3a4e1d7f 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -5,7 +5,7 @@ import logging from hashlib import sha256 from typing import Union, Tuple - +from decimal import Decimal import aio_pika from aiohttp.web_response import Response from eth_account import Account @@ -101,9 +101,9 @@ async def get_message_content( return message_dict, int(str(file_size)) -async def _verify_message_signature(pending_message_db: PendingMessageDb, chain_service: ChainService) -> None: +async def _verify_message_signature(pending_message: BasePendingMessage, chain_service: ChainService) -> None: try: - await chain_service.verify_signature(pending_message_db) + await chain_service.verify_signature(pending_message) except InvalidSignature: raise web.HTTPForbidden() @@ -111,13 +111,13 @@ async def _verify_message_signature(pending_message_db: PendingMessageDb, chain_ async def _verify_user_balance(pending_message_db: PendingMessageDb, session: DbSession, size: int) -> None: current_balance = get_total_balance( session=session, address=pending_message_db.sender - ) + ) or Decimal(0) required_balance = (size / MiB) / 3 # Need to merge to get this functions # current_cost_for_user = get_total_cost_for_address( # session=session, address=pending_message_db.sender # ) - if current_balance < required_balance: + if current_balance < Decimal(required_balance): raise web.HTTPPaymentRequired async def _verify_user_file(message : dict, size : int, file_io) -> None: @@ -132,7 +132,7 @@ async def _verify_user_file(message : dict, size : int, file_io) -> None: raise web.HTTPUnprocessableEntity() -async def storage_add_file_with_message(request: web.Request, session: DbSession, chain_service, post, file_io, sync = False): +async def storage_add_file_with_message(request: web.Request, session: DbSession, chain_service, post, file_io, sync): config = get_config_from_request(request) message, size = await get_message_content(post) mq_queue = None @@ -148,7 +148,7 @@ async def storage_add_file_with_message(request: web.Request, session: DbSession obj=pending_store_message, reception_time=dt.datetime.now(), fetched=True ) await _verify_message_signature( - pending_message_db=pending_message_db, chain_service=chain_service + pending_message=valid_message, chain_service=chain_service ) await _verify_user_balance(session=session, pending_message_db=pending_message_db, size=size) await _verify_user_file(message=message, size=size, file_io=file_io) @@ -163,7 +163,7 @@ async def storage_add_file_with_message(request: web.Request, session: DbSession session.add(pending_message_db) session.commit() - if sync: + if sync and mq_queue: mq_message = await mq_read_one_message(mq_queue, 30) if mq_message is None: @@ -193,8 +193,7 @@ async def storage_add_file(request: web.Request): sync_value = post.get("sync") if sync_value is not None: - sync = ast.literal_eval(sync_value) - + sync = True if sync_value == "True" else False with session_factory() as session: file_hash = await storage_service.add_file( session=session, fileobject=file_io, engine=ItemType.storage From fa37f206baeb5992d9696da879f429f9035a26ca Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 17:48:37 +0200 Subject: [PATCH 18/36] some Refactor --- src/aleph/web/controllers/storage.py | 104 +++++++++++++++------------ tests/api/test_storage.py | 24 ++++--- 2 files changed, 73 insertions(+), 55 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index c3a4e1d7f..63408d1fb 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -7,6 +7,7 @@ from typing import Union, Tuple from decimal import Decimal import aio_pika +import pydantic from aiohttp.web_response import Response from eth_account import Account from eth_account.messages import encode_defunct @@ -26,7 +27,11 @@ from aleph.exceptions import AlephStorageException, UnknownHashError from aleph.toolkit import json from aleph.types.db_session import DbSession, DbSessionFactory -from aleph.types.message_status import MessageProcessingStatus, InvalidSignature, InvalidMessageException +from aleph.types.message_status import ( + MessageProcessingStatus, + InvalidSignature, + InvalidMessageException, +) from aleph.utils import run_in_executor, item_type_from_hash from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, @@ -40,7 +45,8 @@ multidict_proxy_to_io, mq_make_aleph_message_topic_queue, processing_status_to_http_status, - mq_read_one_message, validate_message_dict, + mq_read_one_message, + validate_message_dict, ) from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage @@ -85,30 +91,18 @@ async def add_storage_json_controller(request: web.Request): return web.json_response(output) -async def get_message_content( - post_data: MultiDictProxy[Union[str, bytes, FileField]] -) -> Tuple[dict, int]: - message_bytearray = post_data.get("message", b"") - file_size = post_data.get("file_size") or 0 - - if isinstance(message_bytearray, bytearray): - message_string = message_bytearray.decode("utf-8") - message_dict = json.loads(message_string) - message_dict["time"] = float(message_dict["time"]) - else: - message_dict = {} - - return message_dict, int(str(file_size)) - - -async def _verify_message_signature(pending_message: BasePendingMessage, chain_service: ChainService) -> None: +async def _verify_message_signature( + pending_message: BasePendingMessage, chain_service: ChainService +) -> None: try: await chain_service.verify_signature(pending_message) except InvalidSignature: raise web.HTTPForbidden() -async def _verify_user_balance(pending_message_db: PendingMessageDb, session: DbSession, size: int) -> None: +async def _verify_user_balance( + pending_message_db: PendingMessageDb, session: DbSession, size: int +) -> None: current_balance = get_total_balance( session=session, address=pending_message_db.sender ) or Decimal(0) @@ -120,40 +114,52 @@ async def _verify_user_balance(pending_message_db: PendingMessageDb, session: Db if current_balance < Decimal(required_balance): raise web.HTTPPaymentRequired -async def _verify_user_file(message : dict, size : int, file_io) -> None: + +async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: file_io.seek(0) content = file_io.read(size) - item_content = json.loads(message["item_content"]) + item_content = json.loads(message.item_content) actual_item_hash = sha256(content).hexdigest() client_item_hash = item_content["item_hash"] if len(content) > (1000 * MiB): - raise web.HTTPRequestEntityTooLarge(actual_size=len(content), max_size=(1000 * MiB)) + raise web.HTTPRequestEntityTooLarge( + actual_size=len(content), max_size=(1000 * MiB) + ) elif actual_item_hash != client_item_hash: raise web.HTTPUnprocessableEntity() -async def storage_add_file_with_message(request: web.Request, session: DbSession, chain_service, post, file_io, sync): +class StorageMetadata(pydantic.BaseModel): + message: PendingStoreMessage + file_size: int + sync: bool + + +async def storage_add_file_with_message( + request: web.Request, session: DbSession, chain_service, storage_metadata, file_io +): config = get_config_from_request(request) - message, size = await get_message_content(post) mq_queue = None - try: - valid_message: BasePendingMessage = validate_message_dict(message) - except InvalidMessageException: - output = {"status": "rejected"} - return web.json_response(output, status=422) - - pending_store_message = PendingStoreMessage.parse_obj(valid_message) + pending_store_message = PendingStoreMessage.parse_obj(storage_metadata.message) pending_message_db = PendingMessageDb.from_obj( obj=pending_store_message, reception_time=dt.datetime.now(), fetched=True ) await _verify_message_signature( - pending_message=valid_message, chain_service=chain_service + pending_message=storage_metadata.message, chain_service=chain_service + ) + await _verify_user_balance( + session=session, + pending_message_db=pending_message_db, + size=storage_metadata.file_size, + ) + await _verify_user_file( + message=storage_metadata.message, + size=storage_metadata.file_size, + file_io=file_io, ) - await _verify_user_balance(session=session, pending_message_db=pending_message_db, size=size) - await _verify_user_file(message=message, size=size, file_io=file_io) - if sync: + if storage_metadata.sync: mq_channel = await get_mq_channel_from_request(request, logger=logger) mq_queue = await mq_make_aleph_message_topic_queue( channel=mq_channel, @@ -163,7 +169,7 @@ async def storage_add_file_with_message(request: web.Request, session: DbSession session.add(pending_message_db) session.commit() - if sync and mq_queue: + if storage_metadata.sync and mq_queue: mq_message = await mq_read_one_message(mq_queue, 30) if mq_message is None: @@ -185,21 +191,27 @@ async def storage_add_file(request: web.Request): post = await request.post() file_io = multidict_proxy_to_io(post) post = await request.post() - sync = False - - if post.get("message", b"") is None and len(file_io.read()) > (25 * MiB): - raise web.HTTPUnauthorized() + metadata = post.get("metadata", b"") + storage_metadata = None + try: + storage_metadata = StorageMetadata.parse_raw(metadata) + except Exception as e: + if metadata: + raise web.HTTPUnprocessableEntity() + + if metadata is None: + if len(file_io.read()) > (25 * MiB): + raise web.HTTPUnauthorized() file_io.seek(0) - sync_value = post.get("sync") - if sync_value is not None: - sync = True if sync_value == "True" else False with session_factory() as session: file_hash = await storage_service.add_file( session=session, fileobject=file_io, engine=ItemType.storage ) - if post.get("message", b"") is not None and post.get("file_size") is not None: - await storage_add_file_with_message(request, session, chain_service, post, file_io, sync) + if storage_metadata: + await storage_add_file_with_message( + request, session, chain_service, storage_metadata, file_io + ) session.commit() output = {"status": "success", "hash": file_hash} return web.json_response(output) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 425e7d1c9..77d02190f 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -56,13 +56,13 @@ "type": "STORE", "channel": "null", "signature": "0x2b90dcfa8f93506150df275a4fe670e826be0b4b751badd6ec323648a6a738962f47274f71a9939653fb6d49c25055821f547447fb3b33984a579008d93eca431b", - "time": "1692193373.7144432", + "time": 1692193373.7144432, "item_type": "inline", "item_content": '{"address":"0x6dA130FD646f826C1b8080C07448923DF9a79aaA","time":1692193373.714271,"item_type":"storage","item_hash":"0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f","mime_type":"text/plain"}', "item_hash": "8227acbc2f7c43899efd9f63ea9d8119a4cb142f3ba2db5fe499ccfab86dfaed", "content": { "address": "0x6dA130FD646f826C1b8080C07448923DF9a79aaA", - "time": "1692193373.714271", + "time": 1692193373.714271, "item_type": "storage", "item_hash": "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", "mime_type": "text/plain", @@ -169,12 +169,15 @@ async def add_file_with_message( ) session.commit() - json_data = json.dumps(MESSAGE_DICT) form_data = aiohttp.FormData() form_data.add_field("file", file_content) - form_data.add_field("message", json_data, content_type="application/json") - form_data.add_field("file_size", size) + data = { + "message": MESSAGE_DICT, + "file_size": int(size), + "sync": True, + } + form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) assert response.status == error_code, await response.text() @@ -212,13 +215,16 @@ async def add_file_with_message_202( ) session.commit() - json_data = json.dumps(MESSAGE_DICT) form_data = aiohttp.FormData() form_data.add_field("file", file_content) - form_data.add_field("message", json_data, content_type="application/json") - form_data.add_field("file_size", size) - form_data.add_field("sync", "True") + + data = { + "message": MESSAGE_DICT, + "file_size": int(size), + "sync": False, + } + form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) assert response.status == error_code, await response.text() From 79bd1f8fbb816bfbb7abe10d6ab8afe23e58cbf9 Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 17:54:50 +0200 Subject: [PATCH 19/36] =?UTF-8?q?Fix:=20balances=20contr=C3=B4le=20check?= =?UTF-8?q?=20all=20cost=20for=20user?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aleph/web/controllers/storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 63408d1fb..2e6009998 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -22,6 +22,7 @@ from aleph_message.models import ItemType from multidict import MultiDictProxy from aleph.db.accessors.balances import get_total_balance +from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import count_file_pins, get_file from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError @@ -107,11 +108,10 @@ async def _verify_user_balance( session=session, address=pending_message_db.sender ) or Decimal(0) required_balance = (size / MiB) / 3 - # Need to merge to get this functions - # current_cost_for_user = get_total_cost_for_address( - # session=session, address=pending_message_db.sender - # ) - if current_balance < Decimal(required_balance): + current_cost_for_user = get_total_cost_for_address( + session=session, address=pending_message_db.sender + ) + if current_balance < (Decimal(required_balance) + current_cost_for_user): raise web.HTTPPaymentRequired From 76256d0ff82841ec8236e992cbcd400f9fa9d999 Mon Sep 17 00:00:00 2001 From: 1yam Date: Fri, 25 Aug 2023 17:58:03 +0200 Subject: [PATCH 20/36] Fix: mypy error --- src/aleph/web/controllers/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 2e6009998..ab83637cc 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -118,7 +118,9 @@ async def _verify_user_balance( async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: file_io.seek(0) content = file_io.read(size) - item_content = json.loads(message.item_content) + item_content = {} + if message.item_content: + item_content = json.loads(message.item_content) actual_item_hash = sha256(content).hexdigest() client_item_hash = item_content["item_hash"] if len(content) > (1000 * MiB): @@ -194,7 +196,11 @@ async def storage_add_file(request: web.Request): metadata = post.get("metadata", b"") storage_metadata = None try: - storage_metadata = StorageMetadata.parse_raw(metadata) + if isinstance(metadata, FileField): + metadata_content = metadata.file.read() + storage_metadata = StorageMetadata.parse_raw(metadata_content) + else: + storage_metadata = StorageMetadata.parse_raw(metadata) except Exception as e: if metadata: raise web.HTTPUnprocessableEntity() From 553ee315a570bf10b8237634801680b173b74610 Mon Sep 17 00:00:00 2001 From: 1yam <40899431+1yam@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:12:51 +0200 Subject: [PATCH 21/36] Update src/aleph/web/controllers/storage.py Co-authored-by: Olivier Desenfans --- src/aleph/web/controllers/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index ab83637cc..6aababf93 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -112,7 +112,7 @@ async def _verify_user_balance( session=session, address=pending_message_db.sender ) if current_balance < (Decimal(required_balance) + current_cost_for_user): - raise web.HTTPPaymentRequired + raise web.HTTPPaymentRequired() async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: From 68e4dba68882cf994afa88a4749b8ccb9117e85b Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 11:32:10 +0200 Subject: [PATCH 22/36] Fix: add_file functions who did not seek file object --- src/aleph/storage.py | 1 + src/aleph/web/controllers/storage.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aleph/storage.py b/src/aleph/storage.py index d5e73a613..5b5f1044c 100644 --- a/src/aleph/storage.py +++ b/src/aleph/storage.py @@ -273,6 +273,7 @@ async def add_file( elif engine == ItemType.storage: file_content = fileobject.read() file_hash = sha256(file_content).hexdigest() + fileobject.seek(0) else: raise ValueError(f"Unsupported item type: {engine}") diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 6aababf93..b3e598677 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -116,7 +116,6 @@ async def _verify_user_balance( async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: - file_io.seek(0) content = file_io.read(size) item_content = {} if message.item_content: @@ -204,7 +203,6 @@ async def storage_add_file(request: web.Request): except Exception as e: if metadata: raise web.HTTPUnprocessableEntity() - if metadata is None: if len(file_io.read()) > (25 * MiB): raise web.HTTPUnauthorized() From ab95c92a41248af11eb637801876795a723b9ada Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 11:33:09 +0200 Subject: [PATCH 23/36] Fix: conditions --- src/aleph/web/controllers/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index b3e598677..a6f117750 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -170,7 +170,7 @@ async def storage_add_file_with_message( session.add(pending_message_db) session.commit() - if storage_metadata.sync and mq_queue: + if mq_queue: mq_message = await mq_read_one_message(mq_queue, 30) if mq_message is None: From adc24d4442be15f8d130789eb77f6c1920686491 Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 11:50:56 +0200 Subject: [PATCH 24/36] Fix: Last bug & black --- src/aleph/api_entrypoint.py | 2 +- src/aleph/web/controllers/p2p.py | 8 ++++++-- src/aleph/web/controllers/storage.py | 17 +++++++---------- tests/api/test_storage.py | 10 ++++++---- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/aleph/api_entrypoint.py b/src/aleph/api_entrypoint.py index 84bd592de..2582bf206 100644 --- a/src/aleph/api_entrypoint.py +++ b/src/aleph/api_entrypoint.py @@ -74,7 +74,7 @@ async def configure_aiohttp_app( app[APP_STATE_NODE_CACHE] = node_cache app[APP_STATE_STORAGE_SERVICE] = storage_service app[APP_STATE_SESSION_FACTORY] = session_factory - # app[APP_STATE_CHAIN_SERVICE] = chain_service + app[APP_STATE_CHAIN_SERVICE] = chain_service return app diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index 118a0e8f1..0fe2e28fa 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -26,8 +26,12 @@ get_p2p_client_from_request, get_mq_channel_from_request, ) -from aleph.web.controllers.utils import mq_make_aleph_message_topic_queue, processing_status_to_http_status, \ - mq_read_one_message, validate_message_dict +from aleph.web.controllers.utils import ( + mq_make_aleph_message_topic_queue, + processing_status_to_http_status, + mq_read_one_message, + validate_message_dict, +) LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index a6f117750..32dbeaa37 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -173,22 +173,19 @@ async def storage_add_file_with_message( if mq_queue: mq_message = await mq_read_one_message(mq_queue, 30) - if mq_message is None: + if mq_message is None or mq_message.routing_key is None: raise web.HTTPAccepted() - if mq_message.routing_key is not None: - status_str, _item_hash = mq_message.routing_key.split(".") - processing_status = MessageProcessingStatus(status_str) - status_code = processing_status_to_http_status(processing_status) - return web.json_response(status=status_code, text=_item_hash) + + status_str, _item_hash = mq_message.routing_key.split(".") + processing_status = MessageProcessingStatus(status_str) + status_code = processing_status_to_http_status(processing_status) + return web.json_response(status=status_code, text=_item_hash) async def storage_add_file(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) - # TODO : Add chainservice to ccn_api_client to be able to call get_chainservice_from_request - chain_service: ChainService = ChainService( - session_factory=session_factory, storage_service=storage_service - ) + chain_service: ChainService = get_chain_service_from_request(request) post = await request.post() file_io = multidict_proxy_to_io(post) post = await request.post() diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 77d02190f..92c8acce6 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -98,6 +98,11 @@ def api_client(ccn_api_client, mocker): node_cache=mocker.AsyncMock(), ) + ccn_api_client.app["chain_service"] = ChainService( + session_factory=ccn_api_client.app["session_factory"], + storage_service=ccn_api_client.app["storage_service"], + ) + return ccn_api_client @@ -148,7 +153,6 @@ async def add_file_with_message( mocked_queue = mocker.patch( "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" ) - mocker.patch("aleph.web.controllers.storage.get_chain_service_from_request") # Create a mock MQ response object mock_mq_message = mocker.Mock() @@ -197,8 +201,6 @@ async def add_file_with_message_202( mocked_queue = mocker.patch( "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" ) - mocker.patch("aleph.web.controllers.storage.get_chain_service_from_request") - # Create a mock MQ response object mock_mq_message = mocker.Mock() mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" @@ -222,7 +224,7 @@ async def add_file_with_message_202( data = { "message": MESSAGE_DICT, "file_size": int(size), - "sync": False, + "sync": True, } form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) From fa2c19e1528d8531d4709c22df80eaa5e94fda6f Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 13:37:40 +0200 Subject: [PATCH 25/36] =?UTF-8?q?Fix:=20Contr=C3=B4le=20balance=20only=20i?= =?UTF-8?q?f=20file=20>=2025=20MiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aleph/web/controllers/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 32dbeaa37..b3055e280 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -111,7 +111,7 @@ async def _verify_user_balance( current_cost_for_user = get_total_cost_for_address( session=session, address=pending_message_db.sender ) - if current_balance < (Decimal(required_balance) + current_cost_for_user): + if current_balance < (Decimal(required_balance) + current_cost_for_user) and size > 25 * MiB: raise web.HTTPPaymentRequired() From ae73641a275d035cf780f6643807e15155c3507c Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 13:42:15 +0200 Subject: [PATCH 26/36] Fix: conditions for balance control --- src/aleph/web/controllers/storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index b3055e280..508b7e0cd 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -111,8 +111,9 @@ async def _verify_user_balance( current_cost_for_user = get_total_cost_for_address( session=session, address=pending_message_db.sender ) - if current_balance < (Decimal(required_balance) + current_cost_for_user) and size > 25 * MiB: - raise web.HTTPPaymentRequired() + if size > 25 * MiB: + if current_balance < (Decimal(required_balance) + current_cost_for_user): + raise web.HTTPPaymentRequired() async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: From fd3e64a6ed0adf039822354299d42c9749640d05 Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 14:27:42 +0200 Subject: [PATCH 27/36] Fix: File < 25 MB so return code will be 200 --- tests/api/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 92c8acce6..1bcfc9b83 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -249,7 +249,7 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): b"Hello Aleph.im\n", "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", "15", - "402", + "200", "0", ), ( From d54959b2d2d08175b318da93e4b9de92830e4fea Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 14:32:36 +0200 Subject: [PATCH 28/36] It's only a test --- src/aleph/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/aleph/__init__.py b/src/aleph/__init__.py index 473ee9f84..1ffde49d7 100644 --- a/src/aleph/__init__.py +++ b/src/aleph/__init__.py @@ -3,17 +3,11 @@ from pkg_resources import get_distribution, DistributionNotFound - -def _get_git_version() -> str: - output = subprocess.check_output(("git", "describe", "--tags")) - return output.decode().strip() - - try: # Change here if project is renamed and does not equal the package name dist_name = __name__ __version__ = get_distribution(dist_name).version except DistributionNotFound: - __version__ = _get_git_version() + __version__ = "1.4" finally: del get_distribution, DistributionNotFound From b8d9b1788754bbb752c461195c42f33ac27326b6 Mon Sep 17 00:00:00 2001 From: 1yam Date: Mon, 28 Aug 2023 14:41:53 +0200 Subject: [PATCH 29/36] Revert "It's only a test" This reverts commit f5c8e60e8a45f1e918c92005047d868ddfb09ed7. --- src/aleph/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/aleph/__init__.py b/src/aleph/__init__.py index 1ffde49d7..473ee9f84 100644 --- a/src/aleph/__init__.py +++ b/src/aleph/__init__.py @@ -3,11 +3,17 @@ from pkg_resources import get_distribution, DistributionNotFound + +def _get_git_version() -> str: + output = subprocess.check_output(("git", "describe", "--tags")) + return output.decode().strip() + + try: # Change here if project is renamed and does not equal the package name dist_name = __name__ __version__ = get_distribution(dist_name).version except DistributionNotFound: - __version__ = "1.4" + __version__ = _get_git_version() finally: del get_distribution, DistributionNotFound From a1dc0ffdf3cee526ef3036968c1230da5e8cae0d Mon Sep 17 00:00:00 2001 From: 1yam Date: Tue, 5 Sep 2023 16:20:54 +0200 Subject: [PATCH 30/36] Refactor: more opti --- src/aleph/storage.py | 20 +++++++++++--- src/aleph/web/controllers/storage.py | 40 +++++++++++++++++----------- src/aleph/web/controllers/utils.py | 8 +++--- tests/api/test_storage.py | 1 + 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/aleph/storage.py b/src/aleph/storage.py index 5b5f1044c..2874dc3e9 100644 --- a/src/aleph/storage.py +++ b/src/aleph/storage.py @@ -5,6 +5,7 @@ import logging from hashlib import sha256 from typing import Any, IO, Optional, cast, Final +from aiohttp import web from aleph_message.models import ItemType @@ -19,12 +20,14 @@ from aleph.services.ipfs.common import get_cid_version from aleph.services.p2p.http import request_hash as p2p_http_request_hash from aleph.services.storage.engine import StorageEngine +from aleph.toolkit.constants import MiB from aleph.types.db_session import DbSession from aleph.types.files import FileType from aleph.utils import get_sha256 from aleph.schemas.pending_messages import ( parse_message, ) + LOGGER = logging.getLogger(__name__) @@ -241,7 +244,9 @@ async def get_json( async def pin_hash(self, chash: str, timeout: int = 30, tries: int = 1): await self.ipfs_service.pin_add(cid=chash, timeout=timeout, tries=tries) - async def add_json(self, session: DbSession, value: Any, engine: ItemType = ItemType.ipfs) -> str: + async def add_json( + self, session: DbSession, value: Any, engine: ItemType = ItemType.ipfs + ) -> str: content = aleph_json.dumps(value) if engine == ItemType.ipfs: @@ -262,7 +267,11 @@ async def add_json(self, session: DbSession, value: Any, engine: ItemType = Item return chash async def add_file( - self, session: DbSession, fileobject: IO, engine: ItemType = ItemType.ipfs + self, + session: DbSession, + fileobject: IO, + engine: ItemType = ItemType.ipfs, + size: int = -1, ) -> str: if engine == ItemType.ipfs: output = await self.ipfs_service.add_file(fileobject) @@ -271,9 +280,12 @@ async def add_file( file_content = fileobject.read() elif engine == ItemType.storage: - file_content = fileobject.read() + file_content = fileobject.read(size) + if len(file_content) > (1000 * MiB): + raise web.HTTPRequestEntityTooLarge( + actual_size=len(file_content), max_size=(1000 * MiB) + ) file_hash = sha256(file_content).hexdigest() - fileobject.seek(0) else: raise ValueError(f"Unsupported item type: {engine}") diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 508b7e0cd..823a74a90 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -4,7 +4,7 @@ import functools import logging from hashlib import sha256 -from typing import Union, Tuple +from typing import Union, Tuple, BinaryIO from decimal import Decimal import aio_pika import pydantic @@ -116,18 +116,14 @@ async def _verify_user_balance( raise web.HTTPPaymentRequired() -async def _verify_user_file(message: PendingStoreMessage, size: int, file_io) -> None: - content = file_io.read(size) +async def _verify_user_file( + message: PendingStoreMessage, size: int, file_io, file_hash +) -> None: item_content = {} if message.item_content: item_content = json.loads(message.item_content) - actual_item_hash = sha256(content).hexdigest() client_item_hash = item_content["item_hash"] - if len(content) > (1000 * MiB): - raise web.HTTPRequestEntityTooLarge( - actual_size=len(content), max_size=(1000 * MiB) - ) - elif actual_item_hash != client_item_hash: + if file_hash != client_item_hash: raise web.HTTPUnprocessableEntity() @@ -138,15 +134,20 @@ class StorageMetadata(pydantic.BaseModel): async def storage_add_file_with_message( - request: web.Request, session: DbSession, chain_service, storage_metadata, file_io + request: web.Request, + session: DbSession, + chain_service: ChainService, + storage_metadata: StorageMetadata, + file_io: BinaryIO, + file_hash: str, ): config = get_config_from_request(request) mq_queue = None - pending_store_message = PendingStoreMessage.parse_obj(storage_metadata.message) pending_message_db = PendingMessageDb.from_obj( - obj=pending_store_message, reception_time=dt.datetime.now(), fetched=True + obj=storage_metadata.message, reception_time=dt.datetime.now(), fetched=True ) + await _verify_message_signature( pending_message=storage_metadata.message, chain_service=chain_service ) @@ -159,6 +160,7 @@ async def storage_add_file_with_message( message=storage_metadata.message, size=storage_metadata.file_size, file_io=file_io, + file_hash=file_hash, ) if storage_metadata.sync: @@ -198,7 +200,7 @@ async def storage_add_file(request: web.Request): storage_metadata = StorageMetadata.parse_raw(metadata_content) else: storage_metadata = StorageMetadata.parse_raw(metadata) - except Exception as e: + except (ValueError, TypeError, UnicodeDecodeError) as e: if metadata: raise web.HTTPUnprocessableEntity() if metadata is None: @@ -206,13 +208,21 @@ async def storage_add_file(request: web.Request): raise web.HTTPUnauthorized() file_io.seek(0) + if storage_metadata.file_size > (1000 * MiB): + raise web.HTTPRequestEntityTooLarge( + actual_size=storage_metadata.file_size, max_size=(1000 * MiB) + ) + with session_factory() as session: file_hash = await storage_service.add_file( - session=session, fileobject=file_io, engine=ItemType.storage + session=session, + fileobject=file_io, + engine=ItemType.storage, + size=storage_metadata.file_size if storage_metadata else -1, ) if storage_metadata: await storage_add_file_with_message( - request, session, chain_service, storage_metadata, file_io + request, session, chain_service, storage_metadata, file_io, file_hash ) session.commit() output = {"status": "success", "hash": file_hash} diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index b03744f49..9ab8884a4 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -157,10 +157,11 @@ async def mq_make_aleph_message_topic_queue( auto_delete=False, ) mq_queue = await channel.declare_queue( - auto_delete=True, exclusive=True, + auto_delete=True, + exclusive=True, # Auto-delete the queue after 30 seconds. This guarantees that queues are deleted even # if a bug makes the consumer crash before cleanup. - arguments={"x-expires": 30000} + arguments={"x-expires": 30000}, ) await mq_queue.bind(mq_message_exchange, routing_key=routing_key) return mq_queue @@ -202,4 +203,5 @@ def validate_message_dict(message_dict: Mapping[str, Any]) -> BasePendingMessage try: return parse_message(message_dict) except InvalidMessageException as e: - raise web.HTTPUnprocessableEntity(body=str(e)) \ No newline at end of file + raise web.HTTPUnprocessableEntity(body=str(e)) + diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 1bcfc9b83..4e9bf89ef 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -181,6 +181,7 @@ async def add_file_with_message( "file_size": int(size), "sync": True, } + print(data) form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) From c7ea9990efef1fd9dda157fec3b40907f251b640 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Wed, 6 Sep 2023 16:52:29 +0200 Subject: [PATCH 31/36] pair programming review --- src/aleph/storage.py | 32 ++--- src/aleph/web/controllers/storage.py | 196 +++++++++++++++------------ src/aleph/web/controllers/utils.py | 22 ++- tests/api/test_storage.py | 29 ++-- 4 files changed, 149 insertions(+), 130 deletions(-) diff --git a/src/aleph/storage.py b/src/aleph/storage.py index 2874dc3e9..983d94366 100644 --- a/src/aleph/storage.py +++ b/src/aleph/storage.py @@ -266,12 +266,19 @@ async def add_json( return chash + async def add_file_content_to_local_storage( + self, session: DbSession, file_content: bytes, file_hash: str + ) -> None: + await self.storage_engine.write(filename=file_hash, content=file_content) + upsert_file( + session=session, + file_hash=file_hash, + size=len(file_content), + file_type=FileType.FILE, + ) + async def add_file( - self, - session: DbSession, - fileobject: IO, - engine: ItemType = ItemType.ipfs, - size: int = -1, + self, session: DbSession, fileobject: IO, engine: ItemType = ItemType.ipfs ) -> str: if engine == ItemType.ipfs: output = await self.ipfs_service.add_file(fileobject) @@ -280,21 +287,14 @@ async def add_file( file_content = fileobject.read() elif engine == ItemType.storage: - file_content = fileobject.read(size) - if len(file_content) > (1000 * MiB): - raise web.HTTPRequestEntityTooLarge( - actual_size=len(file_content), max_size=(1000 * MiB) - ) + file_content = fileobject.read() file_hash = sha256(file_content).hexdigest() + else: raise ValueError(f"Unsupported item type: {engine}") - await self.storage_engine.write(filename=file_hash, content=file_content) - upsert_file( - session=session, - file_hash=file_hash, - size=len(file_content), - file_type=FileType.FILE, + await self.add_file_content_to_local_storage( + session=session, file_content=file_content, file_hash=file_hash ) return file_hash diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 823a74a90..9dc673517 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -4,11 +4,12 @@ import functools import logging from hashlib import sha256 -from typing import Union, Tuple, BinaryIO +from typing import Union, Tuple, BinaryIO, Optional from decimal import Decimal import aio_pika import pydantic from aiohttp.web_response import Response +from configmanager import Config from eth_account import Account from eth_account.messages import encode_defunct from mypy.dmypy_server import MiB @@ -19,21 +20,23 @@ from aiohttp import web from aiohttp.web_request import FileField -from aleph_message.models import ItemType +from aleph_message.models import ItemType, StoreContent from multidict import MultiDictProxy from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import count_file_pins, get_file from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError +from aleph.storage import StorageService from aleph.toolkit import json +from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.message_status import ( MessageProcessingStatus, InvalidSignature, InvalidMessageException, ) -from aleph.utils import run_in_executor, item_type_from_hash +from aleph.utils import run_in_executor, item_type_from_hash, get_sha256 from aleph.web.controllers.app_state_getters import ( get_session_factory_from_request, get_storage_service_from_request, @@ -101,132 +104,149 @@ async def _verify_message_signature( raise web.HTTPForbidden() -async def _verify_user_balance( - pending_message_db: PendingMessageDb, session: DbSession, size: int -) -> None: - current_balance = get_total_balance( - session=session, address=pending_message_db.sender - ) or Decimal(0) +async def _verify_user_balance(session: DbSession, address: str, size: int) -> None: + current_balance = get_total_balance(session=session, address=address) or Decimal(0) required_balance = (size / MiB) / 3 - current_cost_for_user = get_total_cost_for_address( - session=session, address=pending_message_db.sender - ) + current_cost_for_user = get_total_cost_for_address(session=session, address=address) if size > 25 * MiB: if current_balance < (Decimal(required_balance) + current_cost_for_user): raise web.HTTPPaymentRequired() -async def _verify_user_file( - message: PendingStoreMessage, size: int, file_io, file_hash -) -> None: - item_content = {} - if message.item_content: - item_content = json.loads(message.item_content) - client_item_hash = item_content["item_hash"] - if file_hash != client_item_hash: - raise web.HTTPUnprocessableEntity() - - class StorageMetadata(pydantic.BaseModel): message: PendingStoreMessage file_size: int sync: bool -async def storage_add_file_with_message( - request: web.Request, +MAX_UNAUTHORIZED_UPLOAD_FILE_SIZE = 25 * MiB +MAX_UPLOAD_FILE_SIZE = 1000 * MiB + + +async def _check_and_add_file( session: DbSession, chain_service: ChainService, - storage_metadata: StorageMetadata, + storage_service: StorageService, + message: Optional[PendingStoreMessage], + file_size: int, file_io: BinaryIO, - file_hash: str, -): - config = get_config_from_request(request) - mq_queue = None +) -> str: + # Perform authentication and balance checks + if message: + await _verify_message_signature( + pending_message=message, chain_service=chain_service + ) + message_content = StoreContent.parse_raw(message.item_content) - pending_message_db = PendingMessageDb.from_obj( - obj=storage_metadata.message, reception_time=dt.datetime.now(), fetched=True - ) + await _verify_user_balance( + session=session, + address=message_content.address, + size=file_size, + ) - await _verify_message_signature( - pending_message=storage_metadata.message, chain_service=chain_service - ) - await _verify_user_balance( + else: + if file_size > MAX_UNAUTHORIZED_UPLOAD_FILE_SIZE: + raise web.HTTPUnauthorized() + message_content = None + + # TODO: this can still reach 1 GiB in memory. We should look into streaming. + file_content = file_io.read(file_size) + file_hash = get_sha256(file_content) + + if message_content: + if message_content.item_hash != file_hash: + raise web.HTTPUnprocessableEntity( + reason=f"File hash does not match ({file_hash} != {message_content.item_hash})" + ) + + await storage_service.add_file_content_to_local_storage( session=session, - pending_message_db=pending_message_db, - size=storage_metadata.file_size, - ) - await _verify_user_file( - message=storage_metadata.message, - size=storage_metadata.file_size, - file_io=file_io, + file_content=file_content, file_hash=file_hash, ) - if storage_metadata.sync: - mq_channel = await get_mq_channel_from_request(request, logger=logger) - mq_queue = await mq_make_aleph_message_topic_queue( - channel=mq_channel, - config=config, - routing_key=f"*.{pending_message_db.item_hash}", - ) + return file_hash - session.add(pending_message_db) - session.commit() - if mq_queue: - mq_message = await mq_read_one_message(mq_queue, 30) - if mq_message is None or mq_message.routing_key is None: - raise web.HTTPAccepted() +async def _make_mq_queue( + request: web.Request, + sync: bool, + routing_key: Optional[str] = None, +) -> Optional[aio_pika.abc.AbstractQueue]: + if not sync: + return None - status_str, _item_hash = mq_message.routing_key.split(".") - processing_status = MessageProcessingStatus(status_str) - status_code = processing_status_to_http_status(processing_status) - return web.json_response(status=status_code, text=_item_hash) + mq_channel = await get_mq_channel_from_request(request, logger) + config = get_config_from_request(request) + return await mq_make_aleph_message_topic_queue( + channel=mq_channel, config=config, routing_key=routing_key + ) async def storage_add_file(request: web.Request): storage_service = get_storage_service_from_request(request) session_factory = get_session_factory_from_request(request) chain_service: ChainService = get_chain_service_from_request(request) + post = await request.post() file_io = multidict_proxy_to_io(post) - post = await request.post() - metadata = post.get("metadata", b"") - storage_metadata = None - try: - if isinstance(metadata, FileField): - metadata_content = metadata.file.read() - storage_metadata = StorageMetadata.parse_raw(metadata_content) - else: - storage_metadata = StorageMetadata.parse_raw(metadata) - except (ValueError, TypeError, UnicodeDecodeError) as e: - if metadata: - raise web.HTTPUnprocessableEntity() - if metadata is None: - if len(file_io.read()) > (25 * MiB): - raise web.HTTPUnauthorized() - file_io.seek(0) + metadata = post.get("metadata") - if storage_metadata.file_size > (1000 * MiB): + status_code = 200 + + if metadata is None: + # User did not provide a message + try: + content_length_str = post["file"].headers["Content-Length"] + file_size = int(content_length_str) + except (KeyError, ValueError) as e: + raise web.HTTPBadRequest(reason="Missing Content-Length header") from e + + message = None + sync = False + + else: + storage_metadata = StorageMetadata.parse_raw(metadata) + message = storage_metadata.message + file_size = storage_metadata.file_size + sync = storage_metadata.sync + + if file_size > MAX_UPLOAD_FILE_SIZE: raise web.HTTPRequestEntityTooLarge( - actual_size=storage_metadata.file_size, max_size=(1000 * MiB) + actual_size=file_size, max_size=MAX_UPLOAD_FILE_SIZE ) with session_factory() as session: - file_hash = await storage_service.add_file( + file_hash = await _check_and_add_file( session=session, - fileobject=file_io, - engine=ItemType.storage, - size=storage_metadata.file_size if storage_metadata else -1, + chain_service=chain_service, + storage_service=storage_service, + message=message, + file_size=file_size, + file_io=file_io, ) - if storage_metadata: - await storage_add_file_with_message( - request, session, chain_service, storage_metadata, file_io, file_hash - ) session.commit() + + if message: + with session_factory() as session: + pending_message_db = PendingMessageDb.from_obj( + obj=message, reception_time=utc_now() + ) + session.add(pending_message_db) + mq_queue = await _make_mq_queue( + request=request, + routing_key=f"*.{message.item_hash}", + sync=sync, + ) + session.commit() + + if mq_queue: + mq_message = await mq_read_one_message(mq_queue, 30) + if not mq_message: + status_code = 202 + output = {"status": "success", "hash": file_hash} - return web.json_response(output) + return web.json_response(data=output, status=status_code) def assert_file_is_downloadable(session: DbSession, file_hash: str) -> None: diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index 9ab8884a4..1ff85a435 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -2,7 +2,7 @@ import json from io import BytesIO, StringIO from math import ceil -from typing import Optional, Union, IO, Mapping, Any +from typing import Optional, Union, IO, Mapping, Any, overload import aio_pika import aiohttp_jinja2 @@ -19,9 +19,22 @@ LIST_FIELD_SEPARATOR = "," -def multidict_proxy_to_io( - multi_dict: MultiDictProxy[Union[str, bytes, FileField]] -) -> IO: +@overload +def multidict_proxy_to_io(multi_dict: MultiDictProxy[bytes]) -> BytesIO: + ... + + +@overload +def multidict_proxy_to_io(multi_dict: MultiDictProxy[str]) -> StringIO: + ... + + +@overload +def multidict_proxy_to_io(multi_dict: MultiDictProxy[FileField]) -> IO: + ... + + +def multidict_proxy_to_io(multi_dict): file_field = multi_dict["file"] if isinstance(file_field, bytes): return BytesIO(file_field) @@ -204,4 +217,3 @@ def validate_message_dict(message_dict: Mapping[str, Any]) -> BasePendingMessage return parse_message(message_dict) except InvalidMessageException as e: raise web.HTTPUnprocessableEntity(body=str(e)) - diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 4e9bf89ef..8cf4204b4 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -1,35 +1,20 @@ import base64 -import datetime import json -from hashlib import sha256 +from decimal import Decimal from typing import Any import aiohttp import orjson import pytest -from aleph_message.models import ItemHash, MessageType, Chain, ItemType -from configmanager import Config +from aleph_message.models import ItemHash, Chain from aleph.chains.chain_service import ChainService -from aleph.handlers.message_handler import MessageHandler -from aleph.schemas.pending_messages import ( - parse_message, -) -from decimal import Decimal - from aleph.db.accessors.files import get_file -from aleph.db.models import PendingMessageDb, AlephBalanceDb -from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage +from aleph.db.models import AlephBalanceDb from aleph.storage import StorageService -from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.channel import Channel from aleph.types.db_session import DbSessionFactory from aleph.types.files import FileType from in_memory_storage_engine import InMemoryStorageEngine -import datetime as dt - - -from aleph.web.controllers.app_state_getters import get_mq_channel_from_request IPFS_ADD_FILE_URI = "/api/v0/ipfs/add_file" IPFS_ADD_JSON_URI = "/api/v0/ipfs/add_json" @@ -117,7 +102,8 @@ async def add_file( form_data.add_field("file", file_content) post_response = await api_client.post(uri, data=form_data) - assert post_response.status == 200, await post_response.text() + response_text = await post_response.text() + assert post_response.status == 200, await response_text post_response_json = await post_response.json() assert post_response_json["status"] == "success" file_hash = post_response_json["hash"] @@ -125,7 +111,7 @@ async def add_file( # Assert that the file is downloadable get_file_response = await api_client.get(f"{GET_STORAGE_RAW_URI}/{file_hash}") - assert get_file_response.status == 200 + assert get_file_response.status == 200, await get_file_response.text() response_data = await get_file_response.read() # Check that the file appears in the DB @@ -185,7 +171,8 @@ async def add_file_with_message( form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) - assert response.status == error_code, await response.text() + response_text = await response.text() + assert response.status == error_code, response_text async def add_file_with_message_202( From 3d8a782edafddbb7f68e761d44eff838771ab505 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Wed, 6 Sep 2023 16:55:43 +0200 Subject: [PATCH 32/36] mypy fix 1 --- src/aleph/web/controllers/storage.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 9dc673517..24c952d95 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -194,7 +194,16 @@ async def storage_add_file(request: web.Request): status_code = 200 - if metadata is None: + if metadata: + metadata_bytes = ( + metadata.file.read() if isinstance(metadata, FileField) else metadata + ) + storage_metadata = StorageMetadata.parse_raw(metadata_bytes) + message = storage_metadata.message + file_size = storage_metadata.file_size + sync = storage_metadata.sync + + else: # User did not provide a message try: content_length_str = post["file"].headers["Content-Length"] @@ -205,12 +214,6 @@ async def storage_add_file(request: web.Request): message = None sync = False - else: - storage_metadata = StorageMetadata.parse_raw(metadata) - message = storage_metadata.message - file_size = storage_metadata.file_size - sync = storage_metadata.sync - if file_size > MAX_UPLOAD_FILE_SIZE: raise web.HTTPRequestEntityTooLarge( actual_size=file_size, max_size=MAX_UPLOAD_FILE_SIZE From a6335b4f2f3fa097ace47c06189d3bb41c4e63cd Mon Sep 17 00:00:00 2001 From: 1yam Date: Thu, 7 Sep 2023 11:53:08 +0200 Subject: [PATCH 33/36] fix mypy --- src/aleph/web/controllers/storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 24c952d95..53b4678d3 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -206,8 +206,9 @@ async def storage_add_file(request: web.Request): else: # User did not provide a message try: - content_length_str = post["file"].headers["Content-Length"] - file_size = int(content_length_str) + if isinstance(post["file"], FileField): + content_length_str = post["file"].headers["Content-Length"] + file_size = int(content_length_str) except (KeyError, ValueError) as e: raise web.HTTPBadRequest(reason="Missing Content-Length header") from e From 127c4f00e2662f1b37efa6efac6720d8d6dc4084 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Fri, 8 Sep 2023 16:29:40 +0200 Subject: [PATCH 34/36] upload works again --- src/aleph/schemas/pending_messages.py | 5 + src/aleph/web/controllers/ipfs.py | 9 +- src/aleph/web/controllers/storage.py | 131 ++++++++++++++++---------- src/aleph/web/controllers/utils.py | 9 +- tests/api/test_storage.py | 3 +- 5 files changed, 97 insertions(+), 60 deletions(-) diff --git a/src/aleph/schemas/pending_messages.py b/src/aleph/schemas/pending_messages.py index 0665c302c..53fba123a 100644 --- a/src/aleph/schemas/pending_messages.py +++ b/src/aleph/schemas/pending_messages.py @@ -124,6 +124,11 @@ class PendingStoreMessage(BasePendingMessage[Literal[MessageType.store], StoreCo pass +class PendingInlineStoreMessage(PendingStoreMessage): + item_content: str + item_type: Literal[ItemType.inline] + + MESSAGE_TYPE_TO_CLASS = { MessageType.aggregate: PendingAggregateMessage, MessageType.forget: PendingForgetMessage, diff --git a/src/aleph/web/controllers/ipfs.py b/src/aleph/web/controllers/ipfs.py index 425aae0a6..5506cf102 100644 --- a/src/aleph/web/controllers/ipfs.py +++ b/src/aleph/web/controllers/ipfs.py @@ -8,7 +8,7 @@ get_ipfs_service_from_request, get_session_factory_from_request, ) -from aleph.web.controllers.utils import multidict_proxy_to_io +from aleph.web.controllers.utils import file_field_to_io async def ipfs_add_file(request: web.Request): @@ -20,7 +20,12 @@ async def ipfs_add_file(request: web.Request): # No need to pin it here anymore. post = await request.post() - ipfs_add_response = await ipfs_service.add_file(multidict_proxy_to_io(post)) + try: + file_field = post["file"] + except KeyError: + raise web.HTTPUnprocessableEntity(reason="Missing 'file' in multipart form.") + + ipfs_add_response = await ipfs_service.add_file(file_field_to_io(file_field)) cid = ipfs_add_response["Hash"] name = ipfs_add_response["Name"] diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 53b4678d3..5a85bbc3a 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -1,40 +1,29 @@ -import ast import base64 -import datetime as dt -import functools +import base64 import logging -from hashlib import sha256 -from typing import Union, Tuple, BinaryIO, Optional from decimal import Decimal +from typing import Union, Optional, Protocol + import aio_pika import pydantic -from aiohttp.web_response import Response -from configmanager import Config -from eth_account import Account -from eth_account.messages import encode_defunct -from mypy.dmypy_server import MiB -from sqlalchemy.orm import Session - -from aleph.chains.chain_service import ChainService -from aleph.chains.common import get_verification_buffer - from aiohttp import web from aiohttp.web_request import FileField from aleph_message.models import ItemType, StoreContent -from multidict import MultiDictProxy +from mypy.dmypy_server import MiB +from pydantic import ValidationError + +from aleph.chains.chain_service import ChainService from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import count_file_pins, get_file from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError +from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage, PendingInlineStoreMessage from aleph.storage import StorageService -from aleph.toolkit import json from aleph.toolkit.timestamp import utc_now -from aleph.types.db_session import DbSession, DbSessionFactory +from aleph.types.db_session import DbSession from aleph.types.message_status import ( - MessageProcessingStatus, InvalidSignature, - InvalidMessageException, ) from aleph.utils import run_in_executor, item_type_from_hash, get_sha256 from aleph.web.controllers.app_state_getters import ( @@ -44,19 +33,17 @@ get_mq_channel_from_request, get_chain_service_from_request, ) - from aleph.web.controllers.utils import ( - multidict_proxy_to_io, + file_field_to_io, mq_make_aleph_message_topic_queue, - processing_status_to_http_status, mq_read_one_message, - validate_message_dict, ) -from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage logger = logging.getLogger(__name__) -MAX_FILE_SIZE = 100 * 1024 * 1024 +MAX_FILE_SIZE = 100 * MiB +MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE = 25 * MiB +MAX_UPLOAD_FILE_SIZE = 1000 * MiB async def add_ipfs_json_controller(request: web.Request): @@ -114,13 +101,45 @@ async def _verify_user_balance(session: DbSession, address: str, size: int) -> N class StorageMetadata(pydantic.BaseModel): - message: PendingStoreMessage + message: PendingInlineStoreMessage file_size: int sync: bool -MAX_UNAUTHORIZED_UPLOAD_FILE_SIZE = 25 * MiB -MAX_UPLOAD_FILE_SIZE = 1000 * MiB +class UploadedFile(Protocol): + size: int + content: Union[str, bytes] + + +class MultipartUploadedFile(UploadedFile): + def __init__(self, file_field: FileField): + self.file_field = file_field + + try: + content_length_str = file_field.headers["Content-Length"] + self.size = int(content_length_str) + except (KeyError, ValueError): + raise web.HTTPUnprocessableEntity( + reason="Invalid/missing Content-Length header." + ) + self._content = None + + @property + def content(self) -> bytes: + # Only read the stream once + if self._content is None: + self._content = self.file_field.file.read(self.size) + + return self._content + + +class RawUploadedFile(UploadedFile): + def __init__(self, content: Union[bytes, str]): + self.content = content + + @property + def size(self) -> int: + return len(self.content) async def _check_and_add_file( @@ -128,29 +147,31 @@ async def _check_and_add_file( chain_service: ChainService, storage_service: StorageService, message: Optional[PendingStoreMessage], - file_size: int, - file_io: BinaryIO, + file: UploadedFile, ) -> str: # Perform authentication and balance checks if message: await _verify_message_signature( pending_message=message, chain_service=chain_service ) - message_content = StoreContent.parse_raw(message.item_content) + try: + message_content = StoreContent.parse_raw(message.item_content) + except ValidationError as e: + raise web.HTTPUnprocessableEntity( + reason=f"Invalid store message content: {e.json()}" + ) await _verify_user_balance( session=session, address=message_content.address, - size=file_size, + size=file.size, ) else: - if file_size > MAX_UNAUTHORIZED_UPLOAD_FILE_SIZE: - raise web.HTTPUnauthorized() message_content = None # TODO: this can still reach 1 GiB in memory. We should look into streaming. - file_content = file_io.read(file_size) + file_content = file.content file_hash = get_sha256(file_content) if message_content: @@ -189,7 +210,16 @@ async def storage_add_file(request: web.Request): chain_service: ChainService = get_chain_service_from_request(request) post = await request.post() - file_io = multidict_proxy_to_io(post) + try: + file_field = post["file"] + except KeyError: + raise web.HTTPUnprocessableEntity(reason="Missing 'file' in multipart form.") + + if isinstance(file_field, FileField): + uploaded_file = MultipartUploadedFile(file_field) + else: + uploaded_file = RawUploadedFile(file_field) + metadata = post.get("metadata") status_code = 200 @@ -198,26 +228,26 @@ async def storage_add_file(request: web.Request): metadata_bytes = ( metadata.file.read() if isinstance(metadata, FileField) else metadata ) - storage_metadata = StorageMetadata.parse_raw(metadata_bytes) + try: + storage_metadata = StorageMetadata.parse_raw(metadata_bytes) + except ValidationError as e: + raise web.HTTPUnprocessableEntity( + reason=f"Could not decode metadata: {e.json()}" + ) + message = storage_metadata.message - file_size = storage_metadata.file_size sync = storage_metadata.sync + max_upload_size = MAX_UPLOAD_FILE_SIZE else: - # User did not provide a message - try: - if isinstance(post["file"], FileField): - content_length_str = post["file"].headers["Content-Length"] - file_size = int(content_length_str) - except (KeyError, ValueError) as e: - raise web.HTTPBadRequest(reason="Missing Content-Length header") from e - + # User did not provide a message in the `metadata` field message = None sync = False + max_upload_size = MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE - if file_size > MAX_UPLOAD_FILE_SIZE: + if uploaded_file.size > max_upload_size: raise web.HTTPRequestEntityTooLarge( - actual_size=file_size, max_size=MAX_UPLOAD_FILE_SIZE + actual_size=uploaded_file.size, max_size=MAX_UPLOAD_FILE_SIZE ) with session_factory() as session: @@ -226,8 +256,7 @@ async def storage_add_file(request: web.Request): chain_service=chain_service, storage_service=storage_service, message=message, - file_size=file_size, - file_io=file_io, + file=uploaded_file, ) session.commit() diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index 1ff85a435..0575ff2da 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -20,22 +20,21 @@ @overload -def multidict_proxy_to_io(multi_dict: MultiDictProxy[bytes]) -> BytesIO: +def file_field_to_io(multi_dict: bytes) -> BytesIO: ... @overload -def multidict_proxy_to_io(multi_dict: MultiDictProxy[str]) -> StringIO: +def file_field_to_io(multi_dict: str) -> StringIO: ... @overload -def multidict_proxy_to_io(multi_dict: MultiDictProxy[FileField]) -> IO: +def file_field_to_io(multi_dict: FileField) -> BytesIO: ... -def multidict_proxy_to_io(multi_dict): - file_field = multi_dict["file"] +def file_field_to_io(file_field): if isinstance(file_field, bytes): return BytesIO(file_field) elif isinstance(file_field, str): diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 8cf4204b4..87d4082b3 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -1,7 +1,7 @@ import base64 import json from decimal import Decimal -from typing import Any +from typing import Any, Optional import aiohttp import orjson @@ -167,7 +167,6 @@ async def add_file_with_message( "file_size": int(size), "sync": True, } - print(data) form_data.add_field("metadata", json.dumps(data), content_type="application/json") response = await api_client.post(uri, data=form_data) From fd278909587fd495ea59b0fe3352a2bd468918a5 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Fri, 8 Sep 2023 18:32:37 +0200 Subject: [PATCH 35/36] nearly done, need mypy fixes + decide what to do with size test --- src/aleph/web/controllers/p2p.py | 114 ++--------------- src/aleph/web/controllers/storage.py | 42 +++---- src/aleph/web/controllers/utils.py | 178 ++++++++++++++++++++++++++- tests/api/test_p2p.py | 6 +- tests/api/test_storage.py | 65 +++++----- 5 files changed, 237 insertions(+), 168 deletions(-) diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index 0fe2e28fa..4bbb54ac6 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -1,55 +1,32 @@ import asyncio import json import logging -from typing import Dict, cast, Optional, Any, Mapping, List, Union +from typing import Dict, cast, Optional, Any, List, Union -import aio_pika.abc from aiohttp import web from aleph_p2p_client import AlephP2PServiceClient from configmanager import Config from pydantic import BaseModel, Field, ValidationError -import aleph.toolkit.json as aleph_json -from aleph.schemas.pending_messages import parse_message, BasePendingMessage from aleph.services.ipfs import IpfsService from aleph.services.p2p.pubsub import publish as pub_p2p from aleph.toolkit.shield import shielded -from aleph.types.message_status import ( - InvalidMessageException, - MessageStatus, - MessageProcessingStatus, -) from aleph.types.protocol import Protocol from aleph.web.controllers.app_state_getters import ( get_config_from_request, get_ipfs_service_from_request, get_p2p_client_from_request, - get_mq_channel_from_request, ) from aleph.web.controllers.utils import ( - mq_make_aleph_message_topic_queue, - processing_status_to_http_status, - mq_read_one_message, validate_message_dict, + broadcast_and_process_message, + PublicationStatus, + broadcast_status_to_http_status, ) LOGGER = logging.getLogger(__name__) -class PublicationStatus(BaseModel): - status: str - failed: List[Protocol] - - @classmethod - def from_failures(cls, failed_publications: List[Protocol]): - status = { - 0: "success", - 1: "warning", - 2: "error", - }[len(failed_publications)] - return cls(status=status, failed=failed_publications) - - def _validate_request_data(config: Config, request_data: Dict) -> None: """ Validates the content of a JSON pubsub message depending on the channel @@ -145,11 +122,6 @@ class PubMessageRequest(BaseModel): message_dict: Dict[str, Any] = Field(alias="message") -class PubMessageResponse(BaseModel): - publication_status: PublicationStatus - message_status: Optional[MessageStatus] - - @shielded async def pub_message(request: web.Request): try: @@ -161,75 +133,13 @@ async def pub_message(request: web.Request): raise web.HTTPUnprocessableEntity() pending_message = validate_message_dict(request_data.message_dict) - - # In sync mode, wait for a message processing event. We need to create the queue - # before publishing the message on P2P topics in order to guarantee that the event - # will be picked up. - config = get_config_from_request(request) - - if request_data.sync: - mq_channel = await get_mq_channel_from_request(request=request, logger=LOGGER) - mq_queue = await mq_make_aleph_message_topic_queue( - channel=mq_channel, - config=config, - routing_key=f"*.{pending_message.item_hash}", - ) - else: - mq_queue = None - - # We publish the message on P2P topics early, for 3 reasons: - # 1. Just because this node is unable to process the message does not - # necessarily mean the message is incorrect (ex: bug in a new version). - # 2. If the publication fails after the processing, we end up in a situation where - # a message exists without being propagated to the other nodes, ultimately - # causing sync issues on the network. - # 3. The message is currently fed to this node using the P2P service client - # loopback mechanism. - ipfs_service = get_ipfs_service_from_request(request) - p2p_client = get_p2p_client_from_request(request) - - message_topic = config.aleph.queue_topic.value - failed_publications = await _pub_on_p2p_topics( - p2p_client=p2p_client, - ipfs_service=ipfs_service, - topic=message_topic, - payload=aleph_json.dumps(request_data.message_dict), - ) - pub_status = PublicationStatus.from_failures(failed_publications) - if pub_status.status == "error": - return web.json_response( - text=PubMessageResponse( - publication_status=pub_status, message_status=None - ).json(), - status=500, - ) - - status = PubMessageResponse( - publication_status=pub_status, message_status=MessageStatus.PENDING + broadcast_status = await broadcast_and_process_message( + pending_message=pending_message, + message_dict=request_data.message_dict, + sync=request_data.sync, + request=request, + logger=LOGGER, ) - # When publishing in async mode, just respond with 202 (Accepted). - message_accepted_response = web.json_response(text=status.json(), status=202) - if not request_data.sync: - return message_accepted_response - - # Ignore type checking here, we know that mq_queue is set at this point - assert mq_queue is not None - response = await mq_read_one_message(mq_queue, timeout=30) - - # Delete the queue immediately - await mq_queue.delete(if_empty=False) - - # If the message was not processed before the timeout, return a 202. - if response is None: - return message_accepted_response - - routing_key = response.routing_key - assert routing_key is not None # again, for type checking - status_str, _item_hash = routing_key.split(".") - processing_status = MessageProcessingStatus(status_str) - status_code = processing_status_to_http_status(processing_status) - - status.message_status = processing_status.to_message_status() - - return web.json_response(text=status.json(), status=status_code) + status_code = broadcast_status_to_http_status(broadcast_status) + return web.json_response(text=broadcast_status.json(), status=status_code) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 5a85bbc3a..1c1bf5242 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -1,5 +1,4 @@ import base64 -import base64 import logging from decimal import Decimal from typing import Union, Optional, Protocol @@ -16,11 +15,13 @@ from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import count_file_pins, get_file -from aleph.db.models import PendingMessageDb from aleph.exceptions import AlephStorageException, UnknownHashError -from aleph.schemas.pending_messages import BasePendingMessage, PendingStoreMessage, PendingInlineStoreMessage +from aleph.schemas.pending_messages import ( + BasePendingMessage, + PendingStoreMessage, + PendingInlineStoreMessage, +) from aleph.storage import StorageService -from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSession from aleph.types.message_status import ( InvalidSignature, @@ -34,9 +35,9 @@ get_chain_service_from_request, ) from aleph.web.controllers.utils import ( - file_field_to_io, mq_make_aleph_message_topic_queue, - mq_read_one_message, + broadcast_and_process_message, + broadcast_status_to_http_status, ) logger = logging.getLogger(__name__) @@ -107,8 +108,13 @@ class StorageMetadata(pydantic.BaseModel): class UploadedFile(Protocol): - size: int - content: Union[str, bytes] + @property + def size(self) -> int: + ... + + @property + def content(self) -> Union[str, bytes]: + ... class MultipartUploadedFile(UploadedFile): @@ -261,22 +267,10 @@ async def storage_add_file(request: web.Request): session.commit() if message: - with session_factory() as session: - pending_message_db = PendingMessageDb.from_obj( - obj=message, reception_time=utc_now() - ) - session.add(pending_message_db) - mq_queue = await _make_mq_queue( - request=request, - routing_key=f"*.{message.item_hash}", - sync=sync, - ) - session.commit() - - if mq_queue: - mq_message = await mq_read_one_message(mq_queue, 30) - if not mq_message: - status_code = 202 + broadcast_status = await broadcast_and_process_message( + pending_message=message, sync=sync, request=request, logger=logger + ) + status_code = broadcast_status_to_http_status(broadcast_status) output = {"status": "success", "hash": file_hash} return web.json_response(data=output, status=status_code) diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index 0575ff2da..bd69cb301 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -1,18 +1,37 @@ import asyncio import json +import logging from io import BytesIO, StringIO from math import ceil -from typing import Optional, Union, IO, Mapping, Any, overload +from typing import Optional, Any, Mapping, List, Union, Dict +from typing import overload import aio_pika +import aio_pika.abc import aiohttp_jinja2 from aiohttp import web from aiohttp.web_request import FileField +from aleph_p2p_client import AlephP2PServiceClient from configmanager import Config -from multidict import MultiDictProxy - -from aleph.schemas.pending_messages import BasePendingMessage, parse_message -from aleph.types.message_status import MessageProcessingStatus, InvalidMessageException +from pydantic import BaseModel + +import aleph.toolkit.json as aleph_json +from aleph.schemas.pending_messages import parse_message, BasePendingMessage +from aleph.services.ipfs import IpfsService +from aleph.services.p2p.pubsub import publish as pub_p2p +from aleph.toolkit.shield import shielded +from aleph.types.message_status import ( + InvalidMessageException, + MessageStatus, + MessageProcessingStatus, +) +from aleph.types.protocol import Protocol +from aleph.web.controllers.app_state_getters import ( + get_config_from_request, + get_ipfs_service_from_request, + get_p2p_client_from_request, + get_mq_channel_from_request, +) DEFAULT_MESSAGES_PER_PAGE = 20 DEFAULT_PAGE = 1 @@ -189,6 +208,15 @@ def processing_status_to_http_status(status: MessageProcessingStatus) -> int: return mapping[status] +def message_status_to_http_status(status: MessageStatus) -> int: + mapping = { + MessageStatus.PENDING: 202, + MessageStatus.PROCESSED: 200, + MessageStatus.REJECTED: 422, + } + return mapping[status] + + async def mq_read_one_message( mq_queue: aio_pika.abc.AbstractQueue, timeout: float ) -> Optional[aio_pika.abc.AbstractIncomingMessage]: @@ -216,3 +244,143 @@ def validate_message_dict(message_dict: Mapping[str, Any]) -> BasePendingMessage return parse_message(message_dict) except InvalidMessageException as e: raise web.HTTPUnprocessableEntity(body=str(e)) + + +class PublicationStatus(BaseModel): + status: str + failed: List[Protocol] + + @classmethod + def from_failures(cls, failed_publications: List[Protocol]): + status = { + 0: "success", + 1: "warning", + 2: "error", + }[len(failed_publications)] + return cls(status=status, failed=failed_publications) + + +async def pub_on_p2p_topics( + p2p_client: AlephP2PServiceClient, + ipfs_service: Optional[IpfsService], + topic: str, + payload: Union[str, bytes], + logger: logging.Logger, +) -> List[Protocol]: + + failed_publications = [] + + if ipfs_service: + try: + await asyncio.wait_for(ipfs_service.pub(topic, payload), 10) + except Exception: + logger.exception("Can't publish on ipfs") + failed_publications.append(Protocol.IPFS) + + try: + await asyncio.wait_for( + pub_p2p( + p2p_client, + topic, + payload, + loopback=True, + ), + 10, + ) + except Exception: + logger.exception("Can't publish on p2p") + failed_publications.append(Protocol.P2P) + + return failed_publications + + +class BroadcastStatus(BaseModel): + publication_status: PublicationStatus + message_status: Optional[MessageStatus] + + +def broadcast_status_to_http_status(broadcast_status: BroadcastStatus) -> int: + if broadcast_status.publication_status == "error": + return 500 + + message_status = broadcast_status.message_status + # Message status should always be set if the publication succeeded + # TODO: improve typing to make this check useless + assert message_status is not None + return message_status_to_http_status(message_status) + + +@shielded +async def broadcast_and_process_message( + pending_message: BasePendingMessage, + sync: bool, + request: web.Request, + logger: logging.Logger, + message_dict: Optional[Dict[str, Any]], +) -> BroadcastStatus: + # In sync mode, wait for a message processing event. We need to create the queue + # before publishing the message on P2P topics in order to guarantee that the event + # will be picked up. + config = get_config_from_request(request) + + if sync: + mq_channel = await get_mq_channel_from_request(request=request, logger=logger) + mq_queue = await mq_make_aleph_message_topic_queue( + channel=mq_channel, + config=config, + routing_key=f"*.{pending_message.item_hash}", + ) + else: + mq_queue = None + + # We publish the message on P2P topics early, for 3 reasons: + # 1. Just because this node is unable to process the message does not + # necessarily mean the message is incorrect (ex: bug in a new version). + # 2. If the publication fails after the processing, we end up in a situation where + # a message exists without being propagated to the other nodes, ultimately + # causing sync issues on the network. + # 3. The message is currently fed to this node using the P2P service client + # loopback mechanism. + ipfs_service = get_ipfs_service_from_request(request) + p2p_client = get_p2p_client_from_request(request) + + message_topic = config.aleph.queue_topic.value + message_dict = message_dict or pending_message.dict(exclude_none=True) + + failed_publications = await pub_on_p2p_topics( + p2p_client=p2p_client, + ipfs_service=ipfs_service, + topic=message_topic, + payload=aleph_json.dumps(message_dict), + logger=logger, + ) + pub_status = PublicationStatus.from_failures(failed_publications) + if pub_status.status == "error": + return BroadcastStatus(publication_status=pub_status, message_status=None) + + status = BroadcastStatus( + publication_status=pub_status, message_status=MessageStatus.PENDING + ) + + # When publishing in async mode, just respond with 202 (Accepted). + if not sync: + return status + + # Ignore type checking here, we know that mq_queue is set at this point + assert mq_queue is not None + response = await mq_read_one_message(mq_queue, timeout=30) + + # Delete the queue immediately + await mq_queue.delete(if_empty=False) + + # If the message was not processed before the timeout, return a 202. + if response is None: + return status + + routing_key = response.routing_key + assert routing_key is not None # again, for type checking + status_str, _item_hash = routing_key.split(".") + processing_status = MessageProcessingStatus(status_str) + + status.message_status = processing_status.to_message_status() + return status diff --git a/tests/api/test_p2p.py b/tests/api/test_p2p.py index ee352319d..93e97df3b 100644 --- a/tests/api/test_p2p.py +++ b/tests/api/test_p2p.py @@ -69,16 +69,16 @@ async def test_pubsub_pub_errors(ccn_api_client, mock_config: Config): @pytest.mark.asyncio async def test_post_message_sync(ccn_api_client, mocker): # Mock the functions used to create the RabbitMQ queue - mocker.patch("aleph.web.controllers.p2p.get_mq_channel_from_request") + mocker.patch("aleph.web.controllers.utils.get_mq_channel_from_request") mocked_queue = mocker.patch( - "aleph.web.controllers.p2p.mq_make_aleph_message_topic_queue" + "aleph.web.controllers.utils.mq_make_aleph_message_topic_queue" ) # Create a mock MQ response object mock_mq_message = mocker.Mock() mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" mocker.patch( - "aleph.web.controllers.p2p._mq_read_one_message", return_value=mock_mq_message + "aleph.web.controllers.utils.mq_read_one_message", return_value=mock_mq_message ) response = await ccn_api_client.post( diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 87d4082b3..b6ca7802b 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -14,6 +14,8 @@ from aleph.storage import StorageService from aleph.types.db_session import DbSessionFactory from aleph.types.files import FileType +from aleph.types.message_status import MessageStatus +from aleph.web.controllers.utils import BroadcastStatus, PublicationStatus from in_memory_storage_engine import InMemoryStorageEngine IPFS_ADD_FILE_URI = "/api/v0/ipfs/add_file" @@ -130,22 +132,17 @@ async def add_file_with_message( session_factory: DbSessionFactory, uri: str, file_content: bytes, - size: str, + size: Optional[int], error_code: int, balance: int, mocker, ): - mocker.patch("aleph.web.controllers.storage.get_mq_channel_from_request") - mocked_queue = mocker.patch( - "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" - ) - - # Create a mock MQ response object - mock_mq_message = mocker.Mock() - mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" mocker.patch( - "aleph.web.controllers.storage.mq_read_one_message", - return_value=mock_mq_message, + "aleph.web.controllers.storage.broadcast_and_process_message", + return_value=BroadcastStatus( + publication_status=PublicationStatus.from_failures([]), + message_status=MessageStatus.PROCESSED, + ), ) with session_factory() as session: @@ -184,15 +181,13 @@ async def add_file_with_message_202( balance: int, mocker, ): - mocker.patch("aleph.web.controllers.storage.get_mq_channel_from_request") - mocked_queue = mocker.patch( - "aleph.web.controllers.storage.mq_make_aleph_message_topic_queue" + mocker.patch( + "aleph.web.controllers.storage.broadcast_and_process_message", + return_value=BroadcastStatus( + publication_status=PublicationStatus.from_failures([]), + message_status=MessageStatus.PENDING, + ), ) - # Create a mock MQ response object - mock_mq_message = mocker.Mock() - mock_mq_message.routing_key = f"processed.{MESSAGE_DICT['item_hash']}" - mocker.patch("aleph.web.controllers.storage.mq_read_one_message", return_value=None) - with session_factory() as session: session.add( AlephBalanceDb( @@ -232,13 +227,15 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): @pytest.mark.parametrize( "file_content, expected_hash, size, error_code, balance", [ - ( - b"Hello Aleph.im\n", - "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - "15", - "200", - "0", - ), + # ( + # b"Hello Aleph.im\n", + # "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + # None, + # "200", + # "0", + # ), + # Invalid size: the advertised size is lower than the size of the file. + # The server should detect this as an invalid file hash and discard the request. ( b"Hello Aleph.im\n", "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", @@ -246,13 +243,13 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): "422", "1000", ), - ( - b"Hello Aleph.im\n", - "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - "15", - "200", - "1000", - ), + # ( + # b"Hello Aleph.im\n", + # "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + # None, + # "200", + # "1000", + # ), ], ) @pytest.mark.asyncio @@ -261,7 +258,7 @@ async def test_storage_add_file_with_message( session_factory: DbSessionFactory, file_content, expected_hash, - size, + size: Optional[int], error_code, balance, mocker, From ad2283fe8414783d8a2d7eb73e3d7dda7d8155e0 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Mon, 11 Sep 2023 23:21:27 +0200 Subject: [PATCH 36/36] fix mypy and remove size test --- src/aleph/schemas/pending_messages.py | 2 +- src/aleph/web/controllers/storage.py | 14 +++++++----- tests/api/test_storage.py | 31 +++++++++------------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/aleph/schemas/pending_messages.py b/src/aleph/schemas/pending_messages.py index 53fba123a..e8203eaf0 100644 --- a/src/aleph/schemas/pending_messages.py +++ b/src/aleph/schemas/pending_messages.py @@ -126,7 +126,7 @@ class PendingStoreMessage(BasePendingMessage[Literal[MessageType.store], StoreCo class PendingInlineStoreMessage(PendingStoreMessage): item_content: str - item_type: Literal[ItemType.inline] + item_type: Literal[ItemType.inline] # type: ignore[valid-type] MESSAGE_TYPE_TO_CLASS = { diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index 1c1bf5242..cc16c42a7 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -103,7 +103,6 @@ async def _verify_user_balance(session: DbSession, address: str, size: int) -> N class StorageMetadata(pydantic.BaseModel): message: PendingInlineStoreMessage - file_size: int sync: bool @@ -117,7 +116,9 @@ def content(self) -> Union[str, bytes]: ... -class MultipartUploadedFile(UploadedFile): +class MultipartUploadedFile: + _content: Optional[bytes] + def __init__(self, file_field: FileField): self.file_field = file_field @@ -139,7 +140,7 @@ def content(self) -> bytes: return self._content -class RawUploadedFile(UploadedFile): +class RawUploadedFile: def __init__(self, content: Union[bytes, str]): self.content = content @@ -178,6 +179,9 @@ async def _check_and_add_file( # TODO: this can still reach 1 GiB in memory. We should look into streaming. file_content = file.content + file_bytes = ( + file_content.encode("utf-8") if isinstance(file_content, str) else file_content + ) file_hash = get_sha256(file_content) if message_content: @@ -188,7 +192,7 @@ async def _check_and_add_file( await storage_service.add_file_content_to_local_storage( session=session, - file_content=file_content, + file_content=file_bytes, file_hash=file_hash, ) @@ -222,7 +226,7 @@ async def storage_add_file(request: web.Request): raise web.HTTPUnprocessableEntity(reason="Missing 'file' in multipart form.") if isinstance(file_field, FileField): - uploaded_file = MultipartUploadedFile(file_field) + uploaded_file: UploadedFile = MultipartUploadedFile(file_field) else: uploaded_file = RawUploadedFile(file_field) diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index b6ca7802b..9be988f52 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -6,6 +6,7 @@ import aiohttp import orjson import pytest +import requests from aleph_message.models import ItemHash, Chain from aleph.chains.chain_service import ChainService @@ -132,7 +133,6 @@ async def add_file_with_message( session_factory: DbSessionFactory, uri: str, file_content: bytes, - size: Optional[int], error_code: int, balance: int, mocker, @@ -161,7 +161,6 @@ async def add_file_with_message( form_data.add_field("file", file_content) data = { "message": MESSAGE_DICT, - "file_size": int(size), "sync": True, } form_data.add_field("metadata", json.dumps(data), content_type="application/json") @@ -227,29 +226,20 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): @pytest.mark.parametrize( "file_content, expected_hash, size, error_code, balance", [ - # ( - # b"Hello Aleph.im\n", - # "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - # None, - # "200", - # "0", - # ), - # Invalid size: the advertised size is lower than the size of the file. - # The server should detect this as an invalid file hash and discard the request. ( b"Hello Aleph.im\n", "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - "10", - "422", + None, + "200", + "0", + ), + ( + b"Hello Aleph.im\n", + "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", + None, + "200", "1000", ), - # ( - # b"Hello Aleph.im\n", - # "0214e5578f5acb5d36ea62255cbf1157a4bdde7b9612b5db4899b2175e310b6f", - # None, - # "200", - # "1000", - # ), ], ) @pytest.mark.asyncio @@ -268,7 +258,6 @@ async def test_storage_add_file_with_message( session_factory, uri=STORAGE_ADD_FILE_URI, file_content=file_content, - size=size, error_code=int(error_code), balance=int(balance), mocker=mocker,