From eab0fef6f9ea476dc1af0c6a375968c8ae6d0005 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Tue, 5 Nov 2024 14:51:06 +0100 Subject: [PATCH 1/3] ALEPH-275 Get more information on program boot Add a command 'aleph program logs' to retrive program logs and help debugging them --- src/aleph_client/commands/program.py | 63 +++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/aleph_client/commands/program.py b/src/aleph_client/commands/program.py index 0d2111d8..d1b60931 100644 --- a/src/aleph_client/commands/program.py +++ b/src/aleph_client/commands/program.py @@ -2,23 +2,29 @@ import json import logging +import sys from base64 import b16decode, b32encode from collections.abc import Mapping from pathlib import Path -from typing import List, Optional +from typing import Any, List, Optional, Tuple from zipfile import BadZipFile import typer +from aiohttp import ClientResponse +from aiohttp.client import _RequestContextManager from aleph.sdk import AuthenticatedAlephHttpClient from aleph.sdk.account import _load_account +from aleph.sdk.client.vm_client import VmClient from aleph.sdk.conf import settings from aleph.sdk.types import AccountFromPrivateKey, StorageEnum -from aleph_message.models import ProgramMessage, StoreMessage +from aleph_message.models import Chain, ProgramMessage, StoreMessage from aleph_message.models.execution.program import ProgramContent from aleph_message.models.item_hash import ItemHash from aleph_message.status import MessageStatus +from click import echo from aleph_client.commands import help_strings +from aleph_client.commands.instance import sanitize_url from aleph_client.commands.utils import ( get_or_prompt_volumes, input_multiline, @@ -236,3 +242,56 @@ async def unpersist( channel=message.channel, ) typer.echo(f"{message.json(indent=4)}") + + +@app.command() +async def logs( + item_hash: str = typer.Argument(..., help="Item hash of program"), + private_key: Optional[str] = settings.PRIVATE_KEY_STRING, + private_key_file: Optional[Path] = settings.PRIVATE_KEY_FILE, + domain: str = typer.Option(None, help="CRN domain on which the VM is stored or running"), + chain: Chain = typer.Option(None, help=help_strings.ADDRESS_CHAIN), + debug: bool = False, +): + """Display logs for the program. + + Will only show logs frp, one select CRN""" + + setup_logging(debug) + + account = _load_account(private_key, private_key_file, chain=chain) + domain = sanitize_url(domain) + + async with VmClient2(account, domain) as client: + async with await client.operate(vm_id=item_hash, operation="logs.json", method="GET") as response: + + logger.debug("Request %s %s", response.url, response.status) + if response.status != 200: + logger.debug(response) + logger.debug(await response.text()) + + if response.status == 404: + echo(f"Execution not found on this server") + return 1 + elif response.status == 403: + + echo(f"You are not the owner of this VM. Maybe try with another wallet?") + + return 1 + elif response.status != 200: + echo(f"Server error: {response.status}. Please try again latter") + return 1 + echo("Received logs") + log_entries = await response.json() + for log in log_entries: + echo(f'{log["__REALTIME_TIMESTAMP"]}> {log["MESSAGE"]}') + + +class VmClient2(VmClient): + async def operate(self, vm_id: ItemHash, operation: str, method: str = "POST") -> _RequestContextManager: + if not self.pubkey_signature_header: + self.pubkey_signature_header = await self._generate_pubkey_signature_header() + + url, header = await self._generate_header(vm_id=vm_id, operation=operation, method=method) + + return self.session.request(method=method, url=url, headers=header) From d571e5bd2fc7e15d17154767b911106c4db09cdc Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Tue, 5 Nov 2024 15:24:43 +0100 Subject: [PATCH 2/3] Move sanitize_url for mypy --- .../commands/instance/__init__.py | 2 +- src/aleph_client/commands/instance/network.py | 24 +---------------- src/aleph_client/commands/program.py | 2 +- src/aleph_client/utils.py | 27 +++++++++++++++++++ tests/unit/test_instance.py | 7 ++--- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/aleph_client/commands/instance/__init__.py b/src/aleph_client/commands/instance/__init__.py index 9ba71870..af83249e 100644 --- a/src/aleph_client/commands/instance/__init__.py +++ b/src/aleph_client/commands/instance/__init__.py @@ -43,6 +43,7 @@ from rich.prompt import Confirm, Prompt from rich.table import Table from rich.text import Text +from utils import sanitize_url from aleph_client.commands import help_strings from aleph_client.commands.instance.display import CRNTable @@ -50,7 +51,6 @@ fetch_crn_info, fetch_vm_info, find_crn_of_vm, - sanitize_url, ) from aleph_client.commands.instance.superfluid import FlowUpdate, update_flow from aleph_client.commands.node import NodeInfo, _fetch_nodes diff --git a/src/aleph_client/commands/instance/network.py b/src/aleph_client/commands/instance/network.py index 48ae984b..9efeff3c 100644 --- a/src/aleph_client/commands/instance/network.py +++ b/src/aleph_client/commands/instance/network.py @@ -4,7 +4,6 @@ from ipaddress import IPv6Interface from json import JSONDecodeError from typing import Optional -from urllib.parse import ParseResult, urlparse import aiohttp from aleph.sdk import AlephHttpClient @@ -13,6 +12,7 @@ from aleph_message.models.execution.base import PaymentType from aleph_message.models.item_hash import ItemHash from pydantic import ValidationError +from utils import sanitize_url from aleph_client.commands import help_strings from aleph_client.commands.node import NodeInfo, _fetch_nodes @@ -42,28 +42,6 @@ PATH_ABOUT_USAGE_SYSTEM = "/about/usage/system" -def sanitize_url(url: str) -> str: - """Ensure that the URL is valid and not obviously irrelevant. - - Args: - url: URL to sanitize. - Returns: - Sanitized URL. - """ - if not url: - raise aiohttp.InvalidURL("Empty URL") - parsed_url: ParseResult = urlparse(url) - if parsed_url.scheme not in ["http", "https"]: - raise aiohttp.InvalidURL(f"Invalid URL scheme: {parsed_url.scheme}") - if parsed_url.hostname in FORBIDDEN_HOSTS: - logger.debug( - f"Invalid URL {url} hostname {parsed_url.hostname} is in the forbidden host list " - f"({', '.join(FORBIDDEN_HOSTS)})" - ) - raise aiohttp.InvalidURL("Invalid URL host") - return url - - async def fetch_crn_info(node_url: str) -> dict | None: """ Fetches compute node usage information and version. diff --git a/src/aleph_client/commands/program.py b/src/aleph_client/commands/program.py index d1b60931..e01e2336 100644 --- a/src/aleph_client/commands/program.py +++ b/src/aleph_client/commands/program.py @@ -22,9 +22,9 @@ from aleph_message.models.item_hash import ItemHash from aleph_message.status import MessageStatus from click import echo +from utils import sanitize_url from aleph_client.commands import help_strings -from aleph_client.commands.instance import sanitize_url from aleph_client.commands.utils import ( get_or_prompt_volumes, input_multiline, diff --git a/src/aleph_client/utils.py b/src/aleph_client/utils.py index 5ad07b1e..d3377f86 100644 --- a/src/aleph_client/utils.py +++ b/src/aleph_client/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import inspect import logging @@ -7,14 +9,17 @@ from pathlib import Path from shutil import make_archive from typing import List, Optional, Tuple, Type, Union +from urllib.parse import ParseResult, urlparse from zipfile import BadZipFile, ZipFile +import aiohttp import typer from aiohttp import ClientSession from aleph.sdk.conf import MainConfiguration, load_main_configuration, settings from aleph.sdk.types import GenericMessage from aleph_message.models.base import MessageType from aleph_message.models.execution.base import Encoding +from commands.instance.network import FORBIDDEN_HOSTS, logger logger = logging.getLogger(__name__) @@ -130,3 +135,25 @@ async def list_unlinked_keys() -> Tuple[List[Path], Optional[MainConfiguration]] unlinked_keys: List[Path] = [key_file for key_file in all_private_key_files if key_file != active_key_path] return unlinked_keys, config + + +def sanitize_url(url: str) -> str: + """Ensure that the URL is valid and not obviously irrelevant. + + Args: + url: URL to sanitize. + Returns: + Sanitized URL. + """ + if not url: + raise aiohttp.InvalidURL("Empty URL") + parsed_url: ParseResult = urlparse(url) + if parsed_url.scheme not in ["http", "https"]: + raise aiohttp.InvalidURL(f"Invalid URL scheme: {parsed_url.scheme}") + if parsed_url.hostname in FORBIDDEN_HOSTS: + logger.debug( + f"Invalid URL {url} hostname {parsed_url.hostname} is in the forbidden host list " + f"({', '.join(FORBIDDEN_HOSTS)})" + ) + raise aiohttp.InvalidURL("Invalid URL host") + return url diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index cefbe81f..5437fe22 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -13,13 +13,10 @@ from aleph_message.models.execution.environment import CpuProperties from eth_utils.currency import to_wei from multidict import CIMultiDict, CIMultiDictProxy +from utils import sanitize_url from aleph_client.commands.instance import delete -from aleph_client.commands.instance.network import ( - FORBIDDEN_HOSTS, - fetch_crn_info, - sanitize_url, -) +from aleph_client.commands.instance.network import FORBIDDEN_HOSTS, fetch_crn_info from aleph_client.models import ( CoreFrequencies, CpuUsage, From 7ab3c4c579647a9047bf45d3fd528f87be38f1f3 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Mon, 18 Nov 2024 10:04:20 +0100 Subject: [PATCH 3/3] fix imports --- src/aleph_client/commands/instance/__init__.py | 3 +-- src/aleph_client/commands/instance/network.py | 18 +----------------- src/aleph_client/commands/program.py | 3 +-- src/aleph_client/utils.py | 18 +++++++++++++++++- tests/unit/test_instance.py | 6 ++---- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/aleph_client/commands/instance/__init__.py b/src/aleph_client/commands/instance/__init__.py index af83249e..feade4f8 100644 --- a/src/aleph_client/commands/instance/__init__.py +++ b/src/aleph_client/commands/instance/__init__.py @@ -43,7 +43,6 @@ from rich.prompt import Confirm, Prompt from rich.table import Table from rich.text import Text -from utils import sanitize_url from aleph_client.commands import help_strings from aleph_client.commands.instance.display import CRNTable @@ -67,7 +66,7 @@ wait_for_processed_instance, ) from aleph_client.models import CRNInfo -from aleph_client.utils import AsyncTyper +from aleph_client.utils import AsyncTyper, sanitize_url logger = logging.getLogger(__name__) app = AsyncTyper(no_args_is_help=True) diff --git a/src/aleph_client/commands/instance/network.py b/src/aleph_client/commands/instance/network.py index 9efeff3c..6394e7ef 100644 --- a/src/aleph_client/commands/instance/network.py +++ b/src/aleph_client/commands/instance/network.py @@ -12,31 +12,15 @@ from aleph_message.models.execution.base import PaymentType from aleph_message.models.item_hash import ItemHash from pydantic import ValidationError -from utils import sanitize_url from aleph_client.commands import help_strings from aleph_client.commands.node import NodeInfo, _fetch_nodes from aleph_client.commands.utils import safe_getattr from aleph_client.models import MachineUsage -from aleph_client.utils import fetch_json +from aleph_client.utils import AsyncTyper, fetch_json, sanitize_url logger = logging.getLogger(__name__) -# Some users had fun adding URLs that are obviously not CRNs. -# If you work for one of these companies, please send a large check to the Aleph team, -# and we may consider removing your domain from the blacklist. Or just use a subdomain. -FORBIDDEN_HOSTS = [ - "amazon.com", - "apple.com", - "facebook.com", - "google.com", - "google.es", - "microsoft.com", - "openai.com", - "twitter.com", - "x.com", - "youtube.com", -] PATH_STATUS_CONFIG = "/status/config" PATH_ABOUT_USAGE_SYSTEM = "/about/usage/system" diff --git a/src/aleph_client/commands/program.py b/src/aleph_client/commands/program.py index e01e2336..a020ad09 100644 --- a/src/aleph_client/commands/program.py +++ b/src/aleph_client/commands/program.py @@ -22,7 +22,6 @@ from aleph_message.models.item_hash import ItemHash from aleph_message.status import MessageStatus from click import echo -from utils import sanitize_url from aleph_client.commands import help_strings from aleph_client.commands.utils import ( @@ -31,7 +30,7 @@ setup_logging, yes_no_input, ) -from aleph_client.utils import AsyncTyper, create_archive +from aleph_client.utils import AsyncTyper, create_archive, sanitize_url logger = logging.getLogger(__name__) app = AsyncTyper(no_args_is_help=True) diff --git a/src/aleph_client/utils.py b/src/aleph_client/utils.py index d3377f86..b8398490 100644 --- a/src/aleph_client/utils.py +++ b/src/aleph_client/utils.py @@ -19,7 +19,6 @@ from aleph.sdk.types import GenericMessage from aleph_message.models.base import MessageType from aleph_message.models.execution.base import Encoding -from commands.instance.network import FORBIDDEN_HOSTS, logger logger = logging.getLogger(__name__) @@ -137,6 +136,23 @@ async def list_unlinked_keys() -> Tuple[List[Path], Optional[MainConfiguration]] return unlinked_keys, config +# Some users had fun adding URLs that are obviously not CRNs. +# If you work for one of these companies, please send a large check to the Aleph team, +# and we may consider removing your domain from the blacklist. Or just use a subdomain. +FORBIDDEN_HOSTS = [ + "amazon.com", + "apple.com", + "facebook.com", + "google.com", + "google.es", + "microsoft.com", + "openai.com", + "twitter.com", + "x.com", + "youtube.com", +] + + def sanitize_url(url: str) -> str: """Ensure that the URL is valid and not obviously irrelevant. diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index 5437fe22..3d3e44f4 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -1,8 +1,6 @@ from __future__ import annotations from datetime import datetime, timezone -from decimal import Decimal -from typing import cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -13,10 +11,9 @@ from aleph_message.models.execution.environment import CpuProperties from eth_utils.currency import to_wei from multidict import CIMultiDict, CIMultiDictProxy -from utils import sanitize_url from aleph_client.commands.instance import delete -from aleph_client.commands.instance.network import FORBIDDEN_HOSTS, fetch_crn_info +from aleph_client.commands.instance.network import fetch_crn_info from aleph_client.models import ( CoreFrequencies, CpuUsage, @@ -28,6 +25,7 @@ MemoryUsage, UsagePeriod, ) +from aleph_client.utils import FORBIDDEN_HOSTS, sanitize_url def dummy_machine_info() -> MachineInfo: