From 63e80fc690409fe0e7d3159d32052e3dc6b5b1cf Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 17:49:21 +0100 Subject: [PATCH 01/61] update --- .gitignore | 1 + MANIFEST.in | 20 +++++ examples/app_commands/.lightning | 1 + examples/app_commands/app.py | 30 ++++++++ setup.py | 2 +- src/lightning_app/cli/lightning_cli.py | 77 +++++++++++++++++-- src/lightning_app/core/api.py | 46 ++++++++++- src/lightning_app/core/app.py | 32 ++++++++ src/lightning_app/core/flow.py | 42 +++++++--- src/lightning_app/core/queues.py | 10 +++ src/lightning_app/runners/backends/backend.py | 3 +- src/lightning_app/runners/multiprocess.py | 2 + .../tests_app/core/scripts/command_example.py | 22 ++++++ tests/tests_app/core/test_lightning_flow.py | 62 ++++++++++++--- 14 files changed, 316 insertions(+), 34 deletions(-) create mode 100644 examples/app_commands/.lightning create mode 100644 examples/app_commands/app.py create mode 100644 tests/tests_app/core/scripts/command_example.py diff --git a/.gitignore b/.gitignore index 47b9bfff92523..c27a9e898de68 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,4 @@ cifar-10-batches-py # ctags tags .tags +src/lightning_app/ui/* diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..b2c6bd31d5624 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,23 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt diff --git a/examples/app_commands/.lightning b/examples/app_commands/.lightning new file mode 100644 index 0000000000000..e198944084960 --- /dev/null +++ b/examples/app_commands/.lightning @@ -0,0 +1 @@ +name: airborne-elbakyan-3490 diff --git a/examples/app_commands/app.py b/examples/app_commands/app.py new file mode 100644 index 0000000000000..4f3de3c355522 --- /dev/null +++ b/examples/app_commands/app.py @@ -0,0 +1,30 @@ +from lightning import LightningFlow +from lightning_app.core.app import LightningApp + + +class ChildFlow(LightningFlow): + def trigger_method(self, name: str): + print(name) + + def configure_commands(self): + return [{"nested_trigger_command": self.trigger_method}] + + +class FlowCommands(LightningFlow): + def __init__(self): + super().__init__() + self.names = [] + self.child_flow = ChildFlow() + + def run(self): + if len(self.names): + print(self.names) + + def trigger_method(self, name: str): + self.names.append(name) + + def configure_commands(self): + return [{"flow_trigger_command": self.trigger_method}] + self.child_flow.configure_commands() + + +app = LightningApp(FlowCommands()) diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index fb39f743ec3a2..62b65aca06adf 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -1,9 +1,10 @@ import logging import os from pathlib import Path -from typing import Tuple, Union +from typing import List, Optional, Tuple, Union import click +import requests from requests.exceptions import ConnectionError from lightning_app import __version__ as ver @@ -14,6 +15,7 @@ from lightning_app.utilities.cli_helpers import _format_input_env_variables from lightning_app.utilities.install_components import register_all_external_components from lightning_app.utilities.login import Auth +from lightning_app.utilities.state import headers_for logger = logging.getLogger(__name__) @@ -116,6 +118,73 @@ def run_app( _run_app(file, cloud, without_server, no_cache, name, blocking, open_ui, env) +@main.group() +def exec(): + """exec your application.""" + + +def retrieve_application_url(app_id_or_name: Optional[str]): + try: + url = "http://127.0.0.1:7501" + response = requests.get(f"{url}/api/v1/commands") + assert response.status_code == 200 + return url, response.json() + except ConnectionError: + from lightning_app.utilities.cloud import _get_project + from lightning_app.utilities.network import LightningClient + + client = LightningClient() + project = _get_project(client) + list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) + + if not app_id_or_name: + raise Exception("Provide an application name or id with --app_id_or_name ...") + + for lightningapp in list_lightningapps.lightningapps: + if lightningapp.id == app_id_or_name or lightningapp.name == app_id_or_name: + response = requests.get(lightningapp.status.url + "/api/v1/commands") + assert response.status_code == 200 + return lightningapp.status.url, response.json() + return None, None + + +@exec.command("app") +@click.argument("command", type=str, default="") +@click.option("--args", type=str, default=[], multiple=True, help="Env variables to be set for the app.") +@click.option("--app_id_or_name", help="The current application name", default="", type=str) +def exec_app( + command: str, + args: List[str], + app_id_or_name: Optional[str] = None, +): + """Run an app from a file.""" + url, commands = retrieve_application_url(app_id_or_name) + if url is None or commands is None: + raise Exception("We couldn't find any matching running app.") + + if not commands: + raise Exception("This application doesn't expose any commands yet.") + + command_names = [c["command"] for c in commands] + if command not in command_names: + raise Exception(f"The provided command {command} isn't available in {command_names}") + + command_metadata = [c for c in commands if c["command"] == command][0] + params = command_metadata["params"] + kwargs = {k.split("=")[0]: k.split("=")[1] for k in args} + for param in params: + if param not in kwargs: + raise Exception(f"The argument --args {param}=X hasn't been provided.") + + json = { + "command_name": command, + "command_arguments": kwargs, + "affiliation": command_metadata["affiliation"], + } + response = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) + assert response.status_code == 200, response.json() + + @main.group(hidden=True) def fork(): """Fork an application.""" @@ -263,10 +332,4 @@ def _prepare_file(file: str) -> str: if exists: return file - if not exists and file == "quick_start.py": - from lightning_app.demo.quick_start import app - - logger.info(f"For demo purposes, Lightning will run the {app.__file__} file.") - return app.__file__ - raise FileNotFoundError(f"The provided file {file} hasn't been found.") diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index 024eb712389b2..96b27efb3a5b9 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -40,6 +40,9 @@ class SessionMiddleware: frontend_static_dir = os.path.join(FRONTEND_DIR, "static") api_app_delta_queue: Queue = None +api_commands_requests_queue: Queue = None +api_commands_metadata_queue: Queue = None + template = {"ui": {}, "app": {}} templates = Jinja2Templates(directory=FRONTEND_DIR) @@ -50,6 +53,7 @@ class SessionMiddleware: lock = Lock() app_spec: Optional[List] = None +app_commands_metadata: Optional[Dict] = None logger = logging.getLogger(__name__) @@ -59,9 +63,10 @@ class SessionMiddleware: class UIRefresher(Thread): - def __init__(self, api_publish_state_queue) -> None: + def __init__(self, api_publish_state_queue, api_commands_metadata_queue) -> None: super().__init__(daemon=True) self.api_publish_state_queue = api_publish_state_queue + self.api_commands_metadata_queue = api_commands_metadata_queue self._exit_event = Event() def run(self): @@ -78,6 +83,14 @@ def run_once(self): except queue.Empty: pass + try: + metadata = self.api_commands_metadata_queue.get(timeout=0) + with lock: + global app_commands_metadata + app_commands_metadata = metadata + except queue.Empty: + pass + def join(self, timeout: Optional[float] = None) -> None: self._exit_event.set() super().join(timeout) @@ -146,6 +159,30 @@ async def get_spec( return app_spec or [] +@fastapi_service.post("/api/v1/commands", response_class=JSONResponse) +async def post_command( + request: Request, +) -> None: + data = await request.json() + command_name = data.get("command_name", None) + if not command_name: + raise Exception("The provided command name is empty.") + command_arguments = data.get("command_arguments", None) + if not command_arguments: + raise Exception("The provided command metadata is empty.") + affiliation = data.get("affiliation", None) + if not affiliation: + raise Exception("The provided affiliation is empty.") + api_commands_requests_queue.put(await request.json()) + + +@fastapi_service.get("/api/v1/commands", response_class=JSONResponse) +async def get_commands() -> Optional[Dict]: + global app_commands_metadata + with lock: + return app_commands_metadata + + @fastapi_service.post("/api/v1/delta") async def post_delta( request: Request, @@ -279,6 +316,8 @@ async def check_is_started(self, queue): def start_server( api_publish_state_queue, api_delta_queue, + commands_requests_queue, + commands_metadata_queue, has_started_queue: Optional[Queue] = None, host="127.0.0.1", port=8000, @@ -288,16 +327,19 @@ def start_server( ): global api_app_delta_queue global global_app_state_store + global api_commands_requests_queue global app_spec app_spec = spec api_app_delta_queue = api_delta_queue + api_commands_requests_queue = commands_requests_queue + api_commands_metadata_queue = commands_metadata_queue if app_state_store is not None: global_app_state_store = app_state_store global_app_state_store.add(TEST_SESSION_UUID) - refresher = UIRefresher(api_publish_state_queue) + refresher = UIRefresher(api_publish_state_queue, api_commands_metadata_queue) refresher.setDaemon(True) refresher.start() diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 81a1a2115e523..bfc84c7fa53d8 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -1,3 +1,4 @@ +import inspect import logging import os import pickle @@ -72,6 +73,8 @@ def __init__( # queues definition. self.delta_queue: t.Optional[BaseQueue] = None self.readiness_queue: t.Optional[BaseQueue] = None + self.commands_requests_queue: t.Optional[BaseQueue] = None + self.commands_metadata_queue: t.Optional[BaseQueue] = None self.api_publish_state_queue: t.Optional[BaseQueue] = None self.api_delta_queue: t.Optional[BaseQueue] = None self.error_queue: t.Optional[BaseQueue] = None @@ -321,6 +324,33 @@ def maybe_apply_changes(self) -> bool: self.set_state(state) self._has_updated = True + def apply_commands(self): + commands = self.root.configure_commands() + commands_metadata = [] + command_names = set() + for command in commands: + for name, method in command.items(): + if name in command_names: + raise Exception(f"The component name {name} has already been used. They need to be unique.") + command_names.add(name) + params = inspect.signature(method).parameters + commands_metadata.append( + { + "command": name, + "affiliation": method.__self__.name, + "params": list(params.keys()), + } + ) + + self.commands_metadata_queue.put(commands_metadata) + + command_query = self.get_state_changed_from_queue(self.commands_requests_queue) + if command_query: + for command in commands: + for command_name, method in command.items(): + if command_query["command_name"] == command_name: + method(**command_query["command_arguments"]) + def run_once(self): """Method used to collect changes and run the root Flow once.""" done = False @@ -345,6 +375,8 @@ def run_once(self): elif self.stage == AppStage.RESTARTING: return self._apply_restarting() + self.apply_commands() + try: self.check_error_queue() t0 = time() diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index a5dcfd0a77e2e..7a1e36492a521 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -79,15 +79,19 @@ def __init__(self): .. doctest:: >>> from lightning import LightningFlow + ... >>> class RootFlow(LightningFlow): + ... ... def __init__(self): ... super().__init__() ... self.counter = 0 + ... ... def run(self): ... self.counter += 1 ... >>> flow = RootFlow() >>> flow.run() + ... >>> assert flow.counter == 1 >>> assert flow.state["vars"]["counter"] == 1 """ @@ -352,12 +356,11 @@ def schedule( from lightning_app import LightningFlow - class Flow(LightningFlow): + def run(self): if self.schedule("hourly"): # run some code once every hour. - print("run this every hour") Arguments: cron_pattern: The cron pattern to provide. Learn more at https://crontab.guru/. @@ -372,8 +375,8 @@ def run(self): from lightning_app import LightningFlow from lightning_app.structures import List - class SchedulerDAG(LightningFlow): + def __init__(self): super().__init__() self.dags = List() @@ -484,10 +487,8 @@ def configure_layout(self) -> Union[Dict[str, Any], List[Dict[str, Any]], Fronte from lightning_app.frontend import StaticWebFrontend - class Flow(LightningFlow): ... - def configure_layout(self): return StaticWebFrontend("path/to/folder/to/serve") @@ -497,19 +498,13 @@ def configure_layout(self): from lightning_app.frontend import StaticWebFrontend - class Flow(LightningFlow): ... - def configure_layout(self): return StreamlitFrontend(render_fn=my_streamlit_ui) - def my_streamlit_ui(state): # add your streamlit code here! - import streamlit as st - - st.button("Hello!") **Example:** Arrange the UI of my children in tabs (default UI by Lightning). @@ -517,11 +512,11 @@ def my_streamlit_ui(state): class Flow(LightningFlow): ... - def configure_layout(self): return [ dict(name="First Tab", content=self.child0), dict(name="Second Tab", content=self.child1), + ... # You can include direct URLs too dict(name="Lightning", content="https://lightning.ai"), ] @@ -608,3 +603,26 @@ def experimental_iterate(self, iterable: Iterable, run_once: bool = True, user_k yield value self._calls[call_hash].update({"has_finished": True}) + + def configure_commands(self): + """Configure the commands of this LightningFlow. + + **Example:** Returns a list of dictionaries mapping a client callback to a flow method. + + .. code-block:: python + + class Flow(LightningFlow): + ... + def __init__(self): + super().__init__() + self.names = [] + + def handle_name_request(name: str) + self.names.append(name) + + def configure_commands(self): + return [ + {"add_name": self.handle_name_request} + ] + """ + raise NotImplementedError diff --git a/src/lightning_app/core/queues.py b/src/lightning_app/core/queues.py index 3b88d896536fe..640d369977d80 100644 --- a/src/lightning_app/core/queues.py +++ b/src/lightning_app/core/queues.py @@ -36,6 +36,8 @@ ORCHESTRATOR_COPY_REQUEST_CONSTANT = "ORCHESTRATOR_COPY_REQUEST" ORCHESTRATOR_COPY_RESPONSE_CONSTANT = "ORCHESTRATOR_COPY_RESPONSE" WORK_QUEUE_CONSTANT = "WORK_QUEUE" +COMMANDS_REQUESTS_QUEUE_CONSTANT = "COMMANDS_REQUESTS_QUEUE" +COMMANDS_METADATA_QUEUE_CONSTANT = "COMMANDS_METADATA_QUEUE" class QueuingSystem(Enum): @@ -51,6 +53,14 @@ def _get_queue(self, queue_name: str) -> "BaseQueue": else: return SingleProcessQueue(queue_name, default_timeout=STATE_UPDATE_TIMEOUT) + def get_commands_requests_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": + queue_name = f"{queue_id}_{COMMANDS_REQUESTS_QUEUE_CONSTANT}" if queue_id else COMMANDS_REQUESTS_QUEUE_CONSTANT + return self._get_queue(queue_name) + + def get_commands_metadata_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": + queue_name = f"{queue_id}_{COMMANDS_METADATA_QUEUE_CONSTANT}" if queue_id else COMMANDS_METADATA_QUEUE_CONSTANT + return self._get_queue(queue_name) + def get_readiness_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": queue_name = f"{queue_id}_{READINESS_QUEUE_CONSTANT}" if queue_id else READINESS_QUEUE_CONSTANT return self._get_queue(queue_name) diff --git a/src/lightning_app/runners/backends/backend.py b/src/lightning_app/runners/backends/backend.py index 80ceb105bbbd1..643f15f0cf6d7 100644 --- a/src/lightning_app/runners/backends/backend.py +++ b/src/lightning_app/runners/backends/backend.py @@ -82,9 +82,10 @@ def _prepare_queues(self, app): kw = dict(queue_id=self.queue_id) app.delta_queue = self.queues.get_delta_queue(**kw) app.readiness_queue = self.queues.get_readiness_queue(**kw) + app.commands_requests_queue = self.queues.get_commands_requests_queue(**kw) + app.commands_metadata_queue = self.queues.get_commands_metadata_queue(**kw) app.error_queue = self.queues.get_error_queue(**kw) app.delta_queue = self.queues.get_delta_queue(**kw) - app.readiness_queue = self.queues.get_readiness_queue(**kw) app.error_queue = self.queues.get_error_queue(**kw) app.api_publish_state_queue = self.queues.get_api_state_publish_queue(**kw) app.api_delta_queue = self.queues.get_api_delta_queue(**kw) diff --git a/src/lightning_app/runners/multiprocess.py b/src/lightning_app/runners/multiprocess.py index 4c58c816c566c..3ec4ebf9206ae 100644 --- a/src/lightning_app/runners/multiprocess.py +++ b/src/lightning_app/runners/multiprocess.py @@ -66,6 +66,8 @@ def dispatch(self, *args: Any, on_before_run: Optional[Callable] = None, **kwarg api_publish_state_queue=self.app.api_publish_state_queue, api_delta_queue=self.app.api_delta_queue, has_started_queue=has_started_queue, + commands_requests_queue=self.app.commands_requests_queue, + commands_metadata_queue=self.app.commands_metadata_queue, spec=extract_metadata_from_app(self.app), ) server_proc = multiprocessing.Process(target=start_server, kwargs=kwargs) diff --git a/tests/tests_app/core/scripts/command_example.py b/tests/tests_app/core/scripts/command_example.py new file mode 100644 index 0000000000000..f9bba430d03be --- /dev/null +++ b/tests/tests_app/core/scripts/command_example.py @@ -0,0 +1,22 @@ +from lightning import LightningFlow +from lightning_app.core.app import LightningApp + + +class FlowCommands(LightningFlow): + def __init__(self): + super().__init__() + self.names = [] + + def run(self): + if len(self.names): + print(self.names) + self._exit() + + def trigger_method(self, name: str): + self.names.append(name) + + def configure_commands(self): + return [{"user_command": self.trigger_method}] + + +app = LightningApp(FlowCommands()) diff --git a/tests/tests_app/core/test_lightning_flow.py b/tests/tests_app/core/test_lightning_flow.py index 26841e057621b..7ac7a72d08313 100644 --- a/tests/tests_app/core/test_lightning_flow.py +++ b/tests/tests_app/core/test_lightning_flow.py @@ -3,21 +3,24 @@ from collections import Counter from copy import deepcopy from dataclasses import dataclass -from time import time +from multiprocessing import Process +from time import sleep, time from unittest.mock import ANY import pytest +from click.testing import CliRunner from deepdiff import DeepDiff, Delta -from lightning_app import LightningApp -from lightning_app.core.flow import LightningFlow -from lightning_app.core.work import LightningWork -from lightning_app.runners import MultiProcessRuntime, SingleProcessRuntime -from lightning_app.storage import Path -from lightning_app.storage.path import storage_root_dir -from lightning_app.testing.helpers import EmptyFlow, EmptyWork -from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef -from lightning_app.utilities.exceptions import ExitAppException +from lightning.app import LightningApp +from lightning.app.cli.lightning_cli import exec_app +from lightning.app.core.flow import LightningFlow +from lightning.app.core.work import LightningWork +from lightning.app.runners import MultiProcessRuntime, SingleProcessRuntime +from lightning.app.storage import Path +from lightning.app.storage.path import storage_root_dir +from lightning.app.testing.helpers import EmptyFlow, EmptyWork +from lightning.app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef +from lightning.app.utilities.exceptions import ExitAppException def test_empty_component(): @@ -543,7 +546,7 @@ def run(self): def test_flow_path_assignment(): - """Test that paths in the lit format lit:// get converted to a proper lightning_app.storage.Path object.""" + """Test that paths in the lit format lit:// get converted to a proper lightning.app.storage.Path object.""" class Flow(LightningFlow): def __init__(self): @@ -635,3 +638,40 @@ def run(self): assert len(self._calls["scheduling"]) == 8 Flow().run() + + +class FlowCommands(LightningFlow): + def __init__(self): + super().__init__() + self.names = [] + + def run(self): + if len(self.names): + print(self.names) + self._exit() + + def trigger_method(self, name: str): + self.names.append(name) + + def configure_commands(self): + return [{"user_command": self.trigger_method}] + + +def target(): + app = LightningApp(FlowCommands()) + MultiProcessRuntime(app).dispatch() + + +def test_configure_commands(): + process = Process(target=target) + process.start() + sleep(5) + runner = CliRunner() + result = runner.invoke( + exec_app, + ["user_command", "--args", "name=something"], + catch_exceptions=False, + ) + sleep(2) + assert result.exit_code == 0 + assert process.exitcode == 0 From 8c940420cb8197c51c0fd3a87e62a28871412190 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 17:51:32 +0100 Subject: [PATCH 02/61] update --- .gitignore | 1 + src/lightning_app/core/flow.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c27a9e898de68..f5c967ad77390 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,4 @@ cifar-10-batches-py tags .tags src/lightning_app/ui/* +examples/template_react_ui/* \ No newline at end of file diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index 7a1e36492a521..e8b9ff55e6c45 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -607,7 +607,7 @@ def experimental_iterate(self, iterable: Iterable, run_once: bool = True, user_k def configure_commands(self): """Configure the commands of this LightningFlow. - **Example:** Returns a list of dictionaries mapping a client callback to a flow method. + Returns a list of dictionaries mapping a command name to a flow method. .. code-block:: python @@ -624,5 +624,18 @@ def configure_commands(self): return [ {"add_name": self.handle_name_request} ] + + Once the app is running with the following command: + + .. code-block:: bash + + lightning run app app.py command + + .. code-block:: bash + + lightning exec app add_name --args name=my_own_name + + + """ raise NotImplementedError From 39e50369c0e1e4d29d0423349baa6412f43a297d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:53:25 +0000 Subject: [PATCH 03/61] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 2 +- src/lightning_app/core/flow.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f5c967ad77390..9574088077f4c 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,4 @@ cifar-10-batches-py tags .tags src/lightning_app/ui/* -examples/template_react_ui/* \ No newline at end of file +examples/template_react_ui/* diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index e8b9ff55e6c45..3f2b044e9ab57 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -634,8 +634,5 @@ def configure_commands(self): .. code-block:: bash lightning exec app add_name --args name=my_own_name - - - """ raise NotImplementedError From 0b6c763a8e78c6f02d94d8e303dd3dc958210aef Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:00:03 +0100 Subject: [PATCH 04/61] update --- MANIFEST.in | 6 ++++++ examples/app_commands/.lightning | 2 +- src/lightning_app/core/flow.py | 3 --- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index b2c6bd31d5624..85d718e3293da 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,3 +23,9 @@ recursive-include src *.md recursive-include requirements *.txt recursive-include src *.md recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt diff --git a/examples/app_commands/.lightning b/examples/app_commands/.lightning index e198944084960..04d8bc7849e19 100644 --- a/examples/app_commands/.lightning +++ b/examples/app_commands/.lightning @@ -1 +1 @@ -name: airborne-elbakyan-3490 +name: commands diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index e8b9ff55e6c45..3f2b044e9ab57 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -634,8 +634,5 @@ def configure_commands(self): .. code-block:: bash lightning exec app add_name --args name=my_own_name - - - """ raise NotImplementedError From 608c4a615cc50a7581a8fd43ec68115f5f5d1d25 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:08:22 +0100 Subject: [PATCH 05/61] update --- MANIFEST.in | 4 ++++ src/lightning_app/core/app.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 85d718e3293da..d559f6c0f22c3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -29,3 +29,7 @@ recursive-include src *.md recursive-include requirements *.txt recursive-include src *.md recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index bfc84c7fa53d8..5d04c00d9c1b0 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -16,7 +16,7 @@ from lightning_app.core.queues import BaseQueue, SingleProcessQueue from lightning_app.frontend import Frontend from lightning_app.storage.path import storage_root_dir -from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef +from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef, is_overridden from lightning_app.utilities.component import _convert_paths_after_init from lightning_app.utilities.enum import AppStage from lightning_app.utilities.exceptions import CacheMissException, ExitAppException @@ -325,6 +325,10 @@ def maybe_apply_changes(self) -> bool: self._has_updated = True def apply_commands(self): + if not is_overridden("configure_commands", self.root): + return + + # Populate commands metadata commands = self.root.configure_commands() commands_metadata = [] command_names = set() @@ -344,6 +348,7 @@ def apply_commands(self): self.commands_metadata_queue.put(commands_metadata) + # Collect requests metadata command_query = self.get_state_changed_from_queue(self.commands_requests_queue) if command_query: for command in commands: From 772d4a574927e0e5712af641d697cadb704ac784 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:24:55 +0100 Subject: [PATCH 06/61] update --- src/lightning_app/CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/lightning_app/CHANGELOG.md diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md new file mode 100644 index 0000000000000..666d9c4912f76 --- /dev/null +++ b/src/lightning_app/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + + +## [0.5.2] - 2022-MM-DD + +### Added + +- Add support for Lightning App Commands through the `configure_commands` hook on the Lightning Flow ([#13602](https://github.com/PyTorchLightning/pytorch-lightning/pull/13602)) + + +### Changed + +### Deprecated + +### Fixed \ No newline at end of file From dca63dec025ee3284e578c8ccd8510ebcea2cdc0 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:25:37 +0100 Subject: [PATCH 07/61] update --- MANIFEST.in | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d559f6c0f22c3..f9f1be1d09e1f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,34 +2,4 @@ exclude *.toml # project config exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py -include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt +include *.cff # citation info \ No newline at end of file From 0091f4ef30db4980d5e1baf87d2ab1ba8a9ad09b Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:25:59 +0100 Subject: [PATCH 08/61] update --- MANIFEST.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f9f1be1d09e1f..bf2f2c7b9706b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ exclude *.toml # project config exclude requirements.txt exclude __pycache__ -include .actions/setup_tools.py -include *.cff # citation info \ No newline at end of file +include .actions/setup_tools.py \ No newline at end of file From ee3f7ad0f972bef69bcb05337e8b2ca6d6087d58 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:26:25 +0100 Subject: [PATCH 09/61] update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ From b06ec89b6591c7b1c883ebe4ec60f0cc8129cc32 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Jul 2022 17:31:54 +0000 Subject: [PATCH 10/61] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- MANIFEST.in | 2 +- src/lightning_app/CHANGELOG.md | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index bf2f2c7b9706b..f388cf0d66a8c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ exclude *.toml # project config exclude requirements.txt exclude __pycache__ -include .actions/setup_tools.py \ No newline at end of file +include .actions/setup_tools.py diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index 666d9c4912f76..b881d12779874 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -4,16 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - -## [0.5.2] - 2022-MM-DD +## \[0.5.2\] - 2022-MM-DD ### Added - Add support for Lightning App Commands through the `configure_commands` hook on the Lightning Flow ([#13602](https://github.com/PyTorchLightning/pytorch-lightning/pull/13602)) - ### Changed ### Deprecated -### Fixed \ No newline at end of file +### Fixed From 59661c2bf9a974a9d1fa4fdf9ab0f065d3fa4fe1 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:35:02 +0100 Subject: [PATCH 11/61] update --- .gitignore | 2 +- tests/tests_app/core/test_lightning_api.py | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9574088077f4c..ad4422b1a7ff7 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,4 @@ cifar-10-batches-py tags .tags src/lightning_app/ui/* -examples/template_react_ui/* +*examples/template_react_ui* diff --git a/tests/tests_app/core/test_lightning_api.py b/tests/tests_app/core/test_lightning_api.py index 81ba6fe0ba179..0173d45fcf9ce 100644 --- a/tests/tests_app/core/test_lightning_api.py +++ b/tests/tests_app/core/test_lightning_api.py @@ -161,10 +161,11 @@ def test_update_publish_state_and_maybe_refresh_ui(): app = AppStageTestingApp(FlowA(), debug=True) publish_state_queue = MockQueue("publish_state_queue") + commands_metadata_queue = MockQueue("commands_metadata_queue") publish_state_queue.put(app.state_with_changes) - thread = UIRefresher(publish_state_queue) + thread = UIRefresher(publish_state_queue, commands_metadata_queue) thread.run_once() assert global_app_state_store.get_app_state("1234") == app.state_with_changes @@ -190,11 +191,19 @@ def get(self, timeout: int = 0): publish_state_queue = InfiniteQueue("publish_state_queue") change_state_queue = MockQueue("change_state_queue") has_started_queue = MockQueue("has_started_queue") + commands_requests_queue = MockQueue("commands_requests_queue") + commands_metadata_queue = MockQueue("commands_metadata_queue") state = app.state_with_changes publish_state_queue.put(state) spec = extract_metadata_from_app(app) ui_refresher = start_server( - publish_state_queue, change_state_queue, has_started_queue=has_started_queue, uvicorn_run=False, spec=spec + publish_state_queue, + change_state_queue, + commands_requests_queue, + commands_metadata_queue, + has_started_queue=has_started_queue, + uvicorn_run=False, + spec=spec, ) headers = headers_for({"type": x_lightning_type}) @@ -331,10 +340,14 @@ def test_start_server_started(): api_publish_state_queue = mp.Queue() api_delta_queue = mp.Queue() has_started_queue = mp.Queue() + commands_requests_queue = mp.Queue() + commands_metadata_queue = mp.Queue() kwargs = dict( api_publish_state_queue=api_publish_state_queue, api_delta_queue=api_delta_queue, has_started_queue=has_started_queue, + commands_requests_queue=commands_requests_queue, + commands_metadata_queue=commands_metadata_queue, port=1111, ) @@ -354,12 +367,16 @@ def test_start_server_info_message(ui_refresher, uvicorn_run, caplog, monkeypatc api_publish_state_queue = MockQueue() api_delta_queue = MockQueue() has_started_queue = MockQueue() + commands_requests_queue = MockQueue() + commands_metadata_queue = MockQueue() kwargs = dict( host=host, port=1111, api_publish_state_queue=api_publish_state_queue, api_delta_queue=api_delta_queue, has_started_queue=has_started_queue, + commands_requests_queue=commands_requests_queue, + commands_metadata_queue=commands_metadata_queue, ) monkeypatch.setattr(api, "logger", logging.getLogger()) From 239a879217b670a824f5223acce2685d2e0df9d0 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:36:03 +0100 Subject: [PATCH 12/61] update --- MANIFEST.in | 1 + value_all | Bin 0 -> 28 bytes value_b | Bin 0 -> 16 bytes value_c | Bin 0 -> 16 bytes 4 files changed, 1 insertion(+) create mode 100644 value_all create mode 100644 value_b create mode 100644 value_c diff --git a/MANIFEST.in b/MANIFEST.in index f388cf0d66a8c..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ exclude *.toml # project config exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py +include *.cff # citation info diff --git a/value_all b/value_all new file mode 100644 index 0000000000000000000000000000000000000000..1de6bc3eb49615e97739125faaa8dd060d2677c3 GIT binary patch literal 28 dcmZo*nJUNt0kKmwdKew2^e{RBvGbHvJpfeS2LJ#7 literal 0 HcmV?d00001 diff --git a/value_b b/value_b new file mode 100644 index 0000000000000000000000000000000000000000..8d6af037feec3cdbdf3678362270d72f19616a7b GIT binary patch literal 16 ScmZo*naaul0X>XPQ}h58j{>j& literal 0 HcmV?d00001 diff --git a/value_c b/value_c new file mode 100644 index 0000000000000000000000000000000000000000..01cf19d810572605e27c594c63551c10bb4365d5 GIT binary patch literal 16 ScmZo*naaul0X>Y)Q}h58k^->+ literal 0 HcmV?d00001 From 855148247864de31a364e99b21c5d97aae4677ae Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:36:28 +0100 Subject: [PATCH 13/61] update --- MANIFEST.in | 2 ++ setup.py | 2 +- value_all | Bin 28 -> 0 bytes value_b | Bin 16 -> 0 bytes value_c | Bin 16 -> 0 bytes 5 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 value_all delete mode 100644 value_b delete mode 100644 value_c diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..ddde80aef9ed8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/value_all b/value_all deleted file mode 100644 index 1de6bc3eb49615e97739125faaa8dd060d2677c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28 dcmZo*nJUNt0kKmwdKew2^e{RBvGbHvJpfeS2LJ#7 diff --git a/value_b b/value_b deleted file mode 100644 index 8d6af037feec3cdbdf3678362270d72f19616a7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 ScmZo*naaul0X>XPQ}h58j{>j& diff --git a/value_c b/value_c deleted file mode 100644 index 01cf19d810572605e27c594c63551c10bb4365d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 ScmZo*naaul0X>Y)Q}h58k^->+ From 215452022ba426bd2935f1f0d8df9fb13dc4a390 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:40:40 +0100 Subject: [PATCH 14/61] update --- src/lightning_app/cli/lightning_cli.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 62b65aca06adf..b63c46cd8e564 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -123,13 +123,19 @@ def exec(): """exec your application.""" -def retrieve_application_url(app_id_or_name: Optional[str]): - try: - url = "http://127.0.0.1:7501" - response = requests.get(f"{url}/api/v1/commands") - assert response.status_code == 200 - return url, response.json() - except ConnectionError: +def _retrieve_application_url(app_id_or_name: Optional[str]): + failed_locally = False + + if app_id_or_name is None: + try: + url = "http://127.0.0.1:7501" + response = requests.get(f"{url}/api/v1/commands") + assert response.status_code == 200 + return url, response.json() + except ConnectionError: + failed_locally = True + + if app_id_or_name or failed_locally: from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.network import LightningClient @@ -137,8 +143,10 @@ def retrieve_application_url(app_id_or_name: Optional[str]): project = _get_project(client) list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) + lightningapp_names = [lightningapp.name for lightningapp in list_lightningapps.lightningapps] + if not app_id_or_name: - raise Exception("Provide an application name or id with --app_id_or_name ...") + raise Exception(f"Provide an application name or id with --app_id_or_name=X. Found {lightningapp_names}") for lightningapp in list_lightningapps.lightningapps: if lightningapp.id == app_id_or_name or lightningapp.name == app_id_or_name: @@ -158,7 +166,7 @@ def exec_app( app_id_or_name: Optional[str] = None, ): """Run an app from a file.""" - url, commands = retrieve_application_url(app_id_or_name) + url, commands = _retrieve_application_url(app_id_or_name) if url is None or commands is None: raise Exception("We couldn't find any matching running app.") From 6c95cf53f688467c8017d18fef75aa2aa50ba0c3 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 18:42:04 +0100 Subject: [PATCH 15/61] update --- src/lightning_app/cli/lightning_cli.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index b63c46cd8e564..811835e49cfe1 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -129,9 +129,10 @@ def _retrieve_application_url(app_id_or_name: Optional[str]): if app_id_or_name is None: try: url = "http://127.0.0.1:7501" - response = requests.get(f"{url}/api/v1/commands") - assert response.status_code == 200 - return url, response.json() + resp = requests.get(f"{url}/api/v1/commands") + if resp.status_code != 200: + raise Exception(f"The server didn't process the request properly. Found {resp.json()}") + return url, resp.json() except ConnectionError: failed_locally = True @@ -150,9 +151,10 @@ def _retrieve_application_url(app_id_or_name: Optional[str]): for lightningapp in list_lightningapps.lightningapps: if lightningapp.id == app_id_or_name or lightningapp.name == app_id_or_name: - response = requests.get(lightningapp.status.url + "/api/v1/commands") - assert response.status_code == 200 - return lightningapp.status.url, response.json() + resp = requests.get(lightningapp.status.url + "/api/v1/commands") + if resp.status_code != 200: + raise Exception(f"The server didn't process the request properly. Found {resp.json()}") + return lightningapp.status.url, resp.json() return None, None @@ -189,8 +191,8 @@ def exec_app( "command_arguments": kwargs, "affiliation": command_metadata["affiliation"], } - response = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) - assert response.status_code == 200, response.json() + resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) + assert resp.status_code == 200, resp.json() @main.group(hidden=True) From 350d11e7a61da030c1db078db368c47ff1eec9f0 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 11 Jul 2022 20:59:39 +0100 Subject: [PATCH 16/61] update --- MANIFEST.in | 2 -- setup.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ddde80aef9ed8..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,3 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ From 2846842d403f6512d3f5ca88b8189b4c51b74524 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Tue, 12 Jul 2022 09:08:44 +0100 Subject: [PATCH 17/61] update --- .actions/delete_cloud_lightning_apps.py | 34 +++++++++++ .actions/download_frontend.py | 5 ++ MANIFEST.in | 14 +++++ examples/app_commands/.lightning | 2 +- setup.py | 2 +- src/lightning_app/cli/lightning_cli.py | 67 ++++++--------------- src/lightning_app/core/app.py | 10 ++- src/lightning_app/core/flow.py | 2 +- src/lightning_app/utilities/cli_helpers.py | 58 +++++++++++++++++- tests/tests_app/core/test_lightning_flow.py | 4 +- 10 files changed, 142 insertions(+), 56 deletions(-) create mode 100644 .actions/delete_cloud_lightning_apps.py create mode 100644 .actions/download_frontend.py diff --git a/.actions/delete_cloud_lightning_apps.py b/.actions/delete_cloud_lightning_apps.py new file mode 100644 index 0000000000000..f682847f1a608 --- /dev/null +++ b/.actions/delete_cloud_lightning_apps.py @@ -0,0 +1,34 @@ +import os + +from lightning_cloud.openapi.rest import ApiException + +from lightning_app.utilities.cloud import _get_project +from lightning_app.utilities.network import LightningClient + +client = LightningClient() + +try: + PR_NUMBER = int(os.getenv("PR_NUMBER", None)) +except (TypeError, ValueError): + # Failed when the PR is running master or 'PR_NUMBER' isn't defined. + PR_NUMBER = "" + +APP_NAME = os.getenv("TEST_APP_NAME", "") + +project = _get_project(client) +list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) + +print([lightningapp.name for lightningapp in list_lightningapps.lightningapps]) + +for lightningapp in list_lightningapps.lightningapps: + if PR_NUMBER and APP_NAME and not lightningapp.name.startswith(f"test-{PR_NUMBER}-{APP_NAME}-"): + continue + print(f"Deleting {lightningapp.name}") + try: + res = client.lightningapp_instance_service_delete_lightningapp_instance( + project_id=project.project_id, + id=lightningapp.id, + ) + assert res == {} + except ApiException as e: + print(f"Failed to delete {lightningapp.name}. Exception {e}") diff --git a/.actions/download_frontend.py b/.actions/download_frontend.py new file mode 100644 index 0000000000000..44d38865803b5 --- /dev/null +++ b/.actions/download_frontend.py @@ -0,0 +1,5 @@ +import lightning_app +from lightning_app.utilities.packaging.lightning_utils import download_frontend + +if __name__ == "__main__": + download_frontend(lightning_app._PROJECT_ROOT) diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..2506b4fa19c0b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,17 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src *.md +recursive-include requirements *.txt diff --git a/examples/app_commands/.lightning b/examples/app_commands/.lightning index 04d8bc7849e19..cebe537814b39 100644 --- a/examples/app_commands/.lightning +++ b/examples/app_commands/.lightning @@ -1 +1 @@ -name: commands +name: test_13 diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 811835e49cfe1..05269f1890d94 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -12,7 +12,10 @@ from lightning_app.core.constants import get_lightning_cloud_url, LOCAL_LAUNCH_ADMIN_VIEW from lightning_app.runners.runtime import dispatch from lightning_app.runners.runtime_type import RuntimeType -from lightning_app.utilities.cli_helpers import _format_input_env_variables +from lightning_app.utilities.cli_helpers import ( + _format_input_env_variables, + _retrieve_application_url_and_available_commands, +) from lightning_app.utilities.install_components import register_all_external_components from lightning_app.utilities.login import Auth from lightning_app.utilities.state import headers_for @@ -118,57 +121,25 @@ def run_app( _run_app(file, cloud, without_server, no_cache, name, blocking, open_ui, env) -@main.group() -def exec(): - """exec your application.""" - - -def _retrieve_application_url(app_id_or_name: Optional[str]): - failed_locally = False - - if app_id_or_name is None: - try: - url = "http://127.0.0.1:7501" - resp = requests.get(f"{url}/api/v1/commands") - if resp.status_code != 200: - raise Exception(f"The server didn't process the request properly. Found {resp.json()}") - return url, resp.json() - except ConnectionError: - failed_locally = True - - if app_id_or_name or failed_locally: - from lightning_app.utilities.cloud import _get_project - from lightning_app.utilities.network import LightningClient - - client = LightningClient() - project = _get_project(client) - list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) - - lightningapp_names = [lightningapp.name for lightningapp in list_lightningapps.lightningapps] - - if not app_id_or_name: - raise Exception(f"Provide an application name or id with --app_id_or_name=X. Found {lightningapp_names}") - - for lightningapp in list_lightningapps.lightningapps: - if lightningapp.id == app_id_or_name or lightningapp.name == app_id_or_name: - resp = requests.get(lightningapp.status.url + "/api/v1/commands") - if resp.status_code != 200: - raise Exception(f"The server didn't process the request properly. Found {resp.json()}") - return lightningapp.status.url, resp.json() - return None, None - - -@exec.command("app") +@run.command("command") @click.argument("command", type=str, default="") -@click.option("--args", type=str, default=[], multiple=True, help="Env variables to be set for the app.") -@click.option("--app_id_or_name", help="The current application name", default="", type=str) -def exec_app( +@click.option( + "--args", + type=str, + default=[], + multiple=True, + help="Arguments to be passed to the method executed in the running app.", +) +@click.option( + "--id", help="Unique identifier for the application. It can be its ID, its url or its name.", default=None, type=str +) +def command( command: str, args: List[str], - app_id_or_name: Optional[str] = None, + id: Optional[str] = None, ): - """Run an app from a file.""" - url, commands = _retrieve_application_url(app_id_or_name) + """Execute a function in a running application from its name.""" + url, commands = _retrieve_application_url_and_available_commands(id) if url is None or commands is None: raise Exception("We couldn't find any matching running app.") diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 5d04c00d9c1b0..a93ed5261cb50 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -325,10 +325,13 @@ def maybe_apply_changes(self) -> bool: self._has_updated = True def apply_commands(self): + """This method is used to apply remotely a collection of commands (methods) from the CLI to a running + app.""" + if not is_overridden("configure_commands", self.root): return - # Populate commands metadata + # 1: Populate commands metadata commands = self.root.configure_commands() commands_metadata = [] command_names = set() @@ -346,14 +349,17 @@ def apply_commands(self): } ) + # 1.2: Pass the collected commands through the queue to the Rest API. self.commands_metadata_queue.put(commands_metadata) - # Collect requests metadata + # 2: Collect requests metadata command_query = self.get_state_changed_from_queue(self.commands_requests_queue) if command_query: for command in commands: for command_name, method in command.items(): if command_query["command_name"] == command_name: + # 2.1: Evaluate the method associated to a specific command. + # Validation is done on the CLI side. method(**command_query["command_arguments"]) def run_once(self): diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index 3f2b044e9ab57..c4295430af797 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -633,6 +633,6 @@ def configure_commands(self): .. code-block:: bash - lightning exec app add_name --args name=my_own_name + lightning run command add_name --args name=my_own_name """ raise NotImplementedError diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index b573440501b3e..c8e8018c49119 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -1,5 +1,11 @@ import re -from typing import Dict +from typing import Dict, Optional + +import requests + +from lightning_app.core.constants import APP_SERVER_PORT +from lightning_app.utilities.cloud import _get_project +from lightning_app.utilities.network import LightningClient def _format_input_env_variables(env_list: tuple) -> Dict[str, str]: @@ -35,3 +41,53 @@ def _format_input_env_variables(env_list: tuple) -> Dict[str, str]: env_vars_dict[var_name] = value return env_vars_dict + + +def _is_url(id: Optional[str]) -> bool: + if isinstance(id, str) and (id.startswith("https://") or id.startswith("http://")): + return True + return False + + +def _retrieve_application_url_and_available_commands(app_id_or_name_or_url: Optional[str]): + """This function is used to retrieve the current url associated with an id.""" + + if _is_url(app_id_or_name_or_url): + url = app_id_or_name_or_url + assert url + resp = requests.get(url + "/api/v1/commands") + if resp.status_code != 200: + raise Exception(f"The server didn't process the request properly. Found {resp.json()}") + return url, resp.json() + + # 2: If no identifier has been provided, evaluate the local application + failed_locally = False + + if app_id_or_name_or_url is None: + try: + url = f"http://127.0.0.1:{APP_SERVER_PORT}" + resp = requests.get(f"{url}/api/v1/commands") + if resp.status_code != 200: + raise Exception(f"The server didn't process the request properly. Found {resp.json()}") + return url, resp.json() + except ConnectionError: + failed_locally = True + + # 3: If an identified was provided or the local evaluation has failed, evaluate the cloud. + if app_id_or_name_or_url or failed_locally: + client = LightningClient() + project = _get_project(client) + list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) + + lightningapp_names = [lightningapp.name for lightningapp in list_lightningapps.lightningapps] + + if not app_id_or_name_or_url: + raise Exception(f"Provide an application name, id or url with --id=X. Found {lightningapp_names}") + + for lightningapp in list_lightningapps.lightningapps: + if lightningapp.id == app_id_or_name_or_url or lightningapp.name == app_id_or_name_or_url: + resp = requests.get(lightningapp.status.url + "/api/v1/commands") + if resp.status_code != 200: + raise Exception(f"The server didn't process the request properly. Found {resp.json()}") + return lightningapp.status.url, resp.json() + return None, None diff --git a/tests/tests_app/core/test_lightning_flow.py b/tests/tests_app/core/test_lightning_flow.py index 7ac7a72d08313..4b38aaa169207 100644 --- a/tests/tests_app/core/test_lightning_flow.py +++ b/tests/tests_app/core/test_lightning_flow.py @@ -12,7 +12,7 @@ from deepdiff import DeepDiff, Delta from lightning.app import LightningApp -from lightning.app.cli.lightning_cli import exec_app +from lightning.app.cli.lightning_cli import command from lightning.app.core.flow import LightningFlow from lightning.app.core.work import LightningWork from lightning.app.runners import MultiProcessRuntime, SingleProcessRuntime @@ -668,7 +668,7 @@ def test_configure_commands(): sleep(5) runner = CliRunner() result = runner.invoke( - exec_app, + command, ["user_command", "--args", "name=something"], catch_exceptions=False, ) From 602de9c2fcae8d166fb0bd9fc28f93ff67adb0d3 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 14 Jul 2022 17:50:04 +0100 Subject: [PATCH 18/61] working poc --- MANIFEST.in | 14 --- src/lightning_app/cli/lightning_cli.py | 30 +++--- src/lightning_app/core/api.py | 3 +- src/lightning_app/core/app.py | 34 +++++-- src/lightning_app/utilities/commands.py | 95 +++++++++++++++++++ .../tests_app/core/scripts/command_example.py | 22 ----- tests/tests_app/core/test_lightning_flow.py | 42 +------- tests/tests_app/utilities/test_commands.py | 68 +++++++++++++ 8 files changed, 210 insertions(+), 98 deletions(-) create mode 100644 src/lightning_app/utilities/commands.py delete mode 100644 tests/tests_app/core/scripts/command_example.py create mode 100644 tests/tests_app/utilities/test_commands.py diff --git a/MANIFEST.in b/MANIFEST.in index 2506b4fa19c0b..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,17 +3,3 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src *.md -recursive-include requirements *.txt diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 05269f1890d94..a99325e360762 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -121,7 +121,7 @@ def run_app( _run_app(file, cloud, without_server, no_cache, name, blocking, open_ui, env) -@run.command("command") +@main.command("command") @click.argument("command", type=str, default="") @click.option( "--args", @@ -139,6 +139,8 @@ def command( id: Optional[str] = None, ): """Execute a function in a running application from its name.""" + from lightning_app.utilities.commands import _download_command + url, commands = _retrieve_application_url_and_available_commands(id) if url is None or commands is None: raise Exception("We couldn't find any matching running app.") @@ -153,17 +155,21 @@ def command( command_metadata = [c for c in commands if c["command"] == command][0] params = command_metadata["params"] kwargs = {k.split("=")[0]: k.split("=")[1] for k in args} - for param in params: - if param not in kwargs: - raise Exception(f"The argument --args {param}=X hasn't been provided.") - - json = { - "command_name": command, - "command_arguments": kwargs, - "affiliation": command_metadata["affiliation"], - } - resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) - assert resp.status_code == 200, resp.json() + if not command_metadata["is_command"]: + for param in params: + if param not in kwargs: + raise Exception(f"The argument --args {param}=X hasn't been provided.") + json = { + "command_name": command, + "command_arguments": kwargs, + "affiliation": command_metadata["affiliation"], + } + resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) + assert resp.status_code == 200, resp.json() + else: + client_command, models = _download_command(command_metadata) + client_command._setup(metadata=command_metadata, models=models, url=url) + client_command.run(**kwargs) @main.group(hidden=True) diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index 96b27efb3a5b9..f4c2c50d252ab 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -173,7 +173,8 @@ async def post_command( affiliation = data.get("affiliation", None) if not affiliation: raise Exception("The provided affiliation is empty.") - api_commands_requests_queue.put(await request.json()) + data = await request.json() + api_commands_requests_queue.put(data) @fastapi_service.get("/api/v1/commands", response_class=JSONResponse) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index a93ed5261cb50..f5e364c05211a 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -3,6 +3,7 @@ import os import pickle import queue +import sys import threading import typing as t import warnings @@ -327,6 +328,7 @@ def maybe_apply_changes(self) -> bool: def apply_commands(self): """This method is used to apply remotely a collection of commands (methods) from the CLI to a running app.""" + from lightning_app.utilities.commands import ClientCommand if not is_overridden("configure_commands", self.root): return @@ -335,17 +337,33 @@ def apply_commands(self): commands = self.root.configure_commands() commands_metadata = [] command_names = set() - for command in commands: - for name, method in command.items(): - if name in command_names: - raise Exception(f"The component name {name} has already been used. They need to be unique.") - command_names.add(name) - params = inspect.signature(method).parameters + for command_mapping in commands: + for command_name, command in command_mapping.items(): + is_command = isinstance(command, ClientCommand) + extra = {} + if is_command: + params = inspect.signature(command.method).parameters + extra = { + "cls_path": inspect.getfile(command.__class__), + "cls_name": command.__class__.__name__, + "params": {p.name: str(p.annotation).split("'")[1].split(".")[-1] for p in params.values()}, + **command._to_dict(), + } + command.models = { + k: getattr(sys.modules[command.__module__], v) for k, v in extra["params"].items() + } + command = command.method + if command_name in command_names: + raise Exception(f"The component name {command_name} has already been used. They need to be unique.") + command_names.add(command_name) + params = inspect.signature(command).parameters commands_metadata.append( { - "command": name, - "affiliation": method.__self__.name, + "command": command_name, + "affiliation": command.__self__.name, "params": list(params.keys()), + "is_command": is_command, + **extra, } ) diff --git a/src/lightning_app/utilities/commands.py b/src/lightning_app/utilities/commands.py new file mode 100644 index 0000000000000..8b4ba7a48624d --- /dev/null +++ b/src/lightning_app/utilities/commands.py @@ -0,0 +1,95 @@ +import errno +import os +import os.path as osp +import shutil +import sys +from getpass import getuser +from importlib.util import module_from_spec, spec_from_file_location +from tempfile import gettempdir +from typing import Any, Callable, Dict, List, Optional, Tuple + +import requests +from pydantic import BaseModel + +from lightning.app.utilities.state import headers_for + + +def makedirs(path: str): + r"""Recursive directory creation function.""" + try: + os.makedirs(osp.expanduser(osp.normpath(path))) + except OSError as e: + if e.errno != errno.EEXIST and osp.isdir(path): + raise e + + +class _Config(BaseModel): + command: str + affiliation: str + params: Dict[str, str] + is_command: bool + cls_path: str + cls_name: str + owner: str + requirements: Optional[List[str]] + + +class ClientCommand: + def __init__(self, method: Callable, requirements: Optional[List[str]] = None) -> None: + self.method = method + flow = getattr(method, "__self__", None) + self.owner = flow.name if flow else None + self.requirements = requirements + self.metadata = None + self.models = Optional[Dict[str, BaseModel]] + self.url = None + + def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], url: str) -> None: + self.metadata = metadata + self.models = models + self.url = url + + def run(self): + """Overrides with the logic to execute on the client side.""" + + def invoke_handler(self, **kwargs: Any) -> Dict[str, Any]: + assert kwargs.keys() == self.models.keys() + for k, v in kwargs.items(): + assert isinstance(v, self.models[k]) + json = { + "command_name": self.metadata["command"], + "command_arguments": {k: v.json() for k, v in kwargs.items()}, + "affiliation": self.metadata["affiliation"], + } + resp = requests.post(self.url + "/api/v1/commands", json=json, headers=headers_for({})) + assert resp.status_code == 200, resp.json() + return resp.json() + + def _to_dict(self): + return {"owner": self.owner, "requirements": self.requirements} + + def __call__(self, **kwargs: Any) -> Any: + assert self.models + kwargs = {k: self.models[k].parse_raw(v) for k, v in kwargs.items()} + return self.method(**kwargs) + + +def _download_command(command_metadata: Dict[str, Any]) -> Tuple[ClientCommand, Dict[str, BaseModel]]: + config = _Config(**command_metadata) + print(config) + if config.cls_path.startswith("s3://"): + raise NotImplementedError() + else: + tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") + makedirs(tmpdir) + cls_name = config.cls_name + target_file = osp.join(tmpdir, f"{config.command}.py") + shutil.copy(config.cls_path, target_file) + spec = spec_from_file_location(config.cls_name, target_file) + mod = module_from_spec(spec) + sys.modules[cls_name] = mod + spec.loader.exec_module(mod) + command = getattr(mod, cls_name)(method=None, requirements=config.requirements) + models = {k: getattr(mod, v) for k, v in config.params.items()} + shutil.rmtree(tmpdir) + return command, models diff --git a/tests/tests_app/core/scripts/command_example.py b/tests/tests_app/core/scripts/command_example.py deleted file mode 100644 index f9bba430d03be..0000000000000 --- a/tests/tests_app/core/scripts/command_example.py +++ /dev/null @@ -1,22 +0,0 @@ -from lightning import LightningFlow -from lightning_app.core.app import LightningApp - - -class FlowCommands(LightningFlow): - def __init__(self): - super().__init__() - self.names = [] - - def run(self): - if len(self.names): - print(self.names) - self._exit() - - def trigger_method(self, name: str): - self.names.append(name) - - def configure_commands(self): - return [{"user_command": self.trigger_method}] - - -app = LightningApp(FlowCommands()) diff --git a/tests/tests_app/core/test_lightning_flow.py b/tests/tests_app/core/test_lightning_flow.py index 4b38aaa169207..74e9688898f45 100644 --- a/tests/tests_app/core/test_lightning_flow.py +++ b/tests/tests_app/core/test_lightning_flow.py @@ -3,16 +3,13 @@ from collections import Counter from copy import deepcopy from dataclasses import dataclass -from multiprocessing import Process -from time import sleep, time +from time import time from unittest.mock import ANY import pytest -from click.testing import CliRunner from deepdiff import DeepDiff, Delta from lightning.app import LightningApp -from lightning.app.cli.lightning_cli import command from lightning.app.core.flow import LightningFlow from lightning.app.core.work import LightningWork from lightning.app.runners import MultiProcessRuntime, SingleProcessRuntime @@ -638,40 +635,3 @@ def run(self): assert len(self._calls["scheduling"]) == 8 Flow().run() - - -class FlowCommands(LightningFlow): - def __init__(self): - super().__init__() - self.names = [] - - def run(self): - if len(self.names): - print(self.names) - self._exit() - - def trigger_method(self, name: str): - self.names.append(name) - - def configure_commands(self): - return [{"user_command": self.trigger_method}] - - -def target(): - app = LightningApp(FlowCommands()) - MultiProcessRuntime(app).dispatch() - - -def test_configure_commands(): - process = Process(target=target) - process.start() - sleep(5) - runner = CliRunner() - result = runner.invoke( - command, - ["user_command", "--args", "name=something"], - catch_exceptions=False, - ) - sleep(2) - assert result.exit_code == 0 - assert process.exitcode == 0 diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py new file mode 100644 index 0000000000000..28ab36fe4659e --- /dev/null +++ b/tests/tests_app/utilities/test_commands.py @@ -0,0 +1,68 @@ +from multiprocessing import Process +from time import sleep + +from click.testing import CliRunner +from pydantic import BaseModel + +from lightning import LightningFlow +from lightning_app import LightningApp +from lightning_app.cli.lightning_cli import command +from lightning_app.runners import MultiProcessRuntime +from lightning_app.utilities.commands import ClientCommand +from lightning_app.utilities.state import AppState + + +class SweepConfig(BaseModel): + sweep_name: str + num_trials: str + + +class SweepCommand(ClientCommand): + def run(self, sweep_name: str, num_trials: str) -> None: + config = SweepConfig(sweep_name=sweep_name, num_trials=num_trials) + _ = self.invoke_handler(config=config) + + +class FlowCommands(LightningFlow): + def __init__(self): + super().__init__() + self.names = [] + + def trigger_method(self, name: str): + self.names.append(name) + + def sweep(self, config: SweepConfig): + print(config) + + def configure_commands(self): + return [{"user_command": self.trigger_method}, {"sweep": SweepCommand(self.sweep)}] + + +def target(): + app = LightningApp(FlowCommands()) + MultiProcessRuntime(app).dispatch() + + +def test_configure_commands(): + process = Process(target=target) + process.start() + sleep(5) + runner = CliRunner() + result = runner.invoke( + command, + ["user_command", "--args", "name=something"], + catch_exceptions=False, + ) + sleep(2) + assert result.exit_code == 0 + state = AppState() + state._request_state() + assert state.names == ["something"] + runner = CliRunner() + result = runner.invoke( + command, + ["sweep", "--args", "sweep_name=my_name", "--args", "num_trials=num_trials"], + catch_exceptions=False, + ) + sleep(2) + assert result.exit_code == 0 From ad5138647a35bd9ce7f2ca507f313c006fcdd030 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 14 Jul 2022 18:05:37 +0100 Subject: [PATCH 19/61] update --- src/lightning_app/cli/lightning_cli.py | 2 +- src/lightning_app/core/app.py | 2 +- src/lightning_app/utilities/commands/__init__.py | 0 src/lightning_app/utilities/{commands.py => commands/base.py} | 0 tests/tests_app/utilities/test_commands.py | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 src/lightning_app/utilities/commands/__init__.py rename src/lightning_app/utilities/{commands.py => commands/base.py} (100%) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index a99325e360762..24b0ea7d96f7e 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -139,7 +139,7 @@ def command( id: Optional[str] = None, ): """Execute a function in a running application from its name.""" - from lightning_app.utilities.commands import _download_command + from lightning_app.utilities.commands.base import _download_command url, commands = _retrieve_application_url_and_available_commands(id) if url is None or commands is None: diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index f5e364c05211a..979605823992b 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -328,7 +328,7 @@ def maybe_apply_changes(self) -> bool: def apply_commands(self): """This method is used to apply remotely a collection of commands (methods) from the CLI to a running app.""" - from lightning_app.utilities.commands import ClientCommand + from lightning_app.utilities.commands.base import ClientCommand if not is_overridden("configure_commands", self.root): return diff --git a/src/lightning_app/utilities/commands/__init__.py b/src/lightning_app/utilities/commands/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/lightning_app/utilities/commands.py b/src/lightning_app/utilities/commands/base.py similarity index 100% rename from src/lightning_app/utilities/commands.py rename to src/lightning_app/utilities/commands/base.py diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 28ab36fe4659e..0e1ec1b3b6f44 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -8,7 +8,7 @@ from lightning_app import LightningApp from lightning_app.cli.lightning_cli import command from lightning_app.runners import MultiProcessRuntime -from lightning_app.utilities.commands import ClientCommand +from lightning_app.utilities.commands.base import ClientCommand from lightning_app.utilities.state import AppState From e678cbadc741cac688bc7fa2f18b2851ccd8cc5d Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 14 Jul 2022 19:17:07 +0100 Subject: [PATCH 20/61] wip --- src/lightning_app/cli/lightning_cli.py | 2 ++ src/lightning_app/core/api.py | 6 ++++++ src/lightning_app/core/app.py | 2 +- src/lightning_app/runners/multiprocess.py | 3 +++ src/lightning_app/utilities/commands/base.py | 2 ++ tests/tests_app/utilities/test_commands.py | 4 +++- 6 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 24b0ea7d96f7e..a4915648cf4be 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -2,6 +2,7 @@ import os from pathlib import Path from typing import List, Optional, Tuple, Union +from uuid import uuid4 import click import requests @@ -163,6 +164,7 @@ def command( "command_name": command, "command_arguments": kwargs, "affiliation": command_metadata["affiliation"], + "id": str(uuid4()), } resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index f4c2c50d252ab..a94524645add8 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -54,6 +54,7 @@ class SessionMiddleware: app_spec: Optional[List] = None app_commands_metadata: Optional[Dict] = None +commands_response_store = {} logger = logging.getLogger(__name__) @@ -174,7 +175,11 @@ async def post_command( if not affiliation: raise Exception("The provided affiliation is empty.") data = await request.json() + # request_id = data["id"] api_commands_requests_queue.put(data) + # response = api_commands_requests_queue.get() + # if request_id == response["response_id"]: + # return response @fastapi_service.get("/api/v1/commands", response_class=JSONResponse) @@ -318,6 +323,7 @@ def start_server( api_publish_state_queue, api_delta_queue, commands_requests_queue, + commands_response_queue, commands_metadata_queue, has_started_queue: Optional[Queue] = None, host="127.0.0.1", diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 979605823992b..8dbd838de3351 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -378,7 +378,7 @@ def apply_commands(self): if command_query["command_name"] == command_name: # 2.1: Evaluate the method associated to a specific command. # Validation is done on the CLI side. - method(**command_query["command_arguments"]) + response = method(**command_query["command_arguments"]) def run_once(self): """Method used to collect changes and run the root Flow once.""" diff --git a/src/lightning_app/runners/multiprocess.py b/src/lightning_app/runners/multiprocess.py index 3ec4ebf9206ae..61274f1b46967 100644 --- a/src/lightning_app/runners/multiprocess.py +++ b/src/lightning_app/runners/multiprocess.py @@ -1,3 +1,4 @@ +import asyncio import multiprocessing import os from dataclasses import dataclass @@ -60,6 +61,7 @@ def dispatch(self, *args: Any, on_before_run: Optional[Callable] = None, **kwarg if self.start_server: self.app.should_publish_changes_to_api = True has_started_queue = self.backend.queues.get_has_server_started_queue() + self.app.commands_response_queue = asyncio.Queue() kwargs = dict( host=self.host, port=self.port, @@ -67,6 +69,7 @@ def dispatch(self, *args: Any, on_before_run: Optional[Callable] = None, **kwarg api_delta_queue=self.app.api_delta_queue, has_started_queue=has_started_queue, commands_requests_queue=self.app.commands_requests_queue, + commands_response_queue=self.app.commands_response_queue, commands_metadata_queue=self.app.commands_metadata_queue, spec=extract_metadata_from_app(self.app), ) diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 8b4ba7a48624d..5d8f934085a44 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -7,6 +7,7 @@ from importlib.util import module_from_spec, spec_from_file_location from tempfile import gettempdir from typing import Any, Callable, Dict, List, Optional, Tuple +from uuid import uuid4 import requests from pydantic import BaseModel @@ -60,6 +61,7 @@ def invoke_handler(self, **kwargs: Any) -> Dict[str, Any]: "command_name": self.metadata["command"], "command_arguments": {k: v.json() for k, v in kwargs.items()}, "affiliation": self.metadata["affiliation"], + "id": str(uuid4()), } resp = requests.post(self.url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 0e1ec1b3b6f44..a892b0494ee7f 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -20,7 +20,8 @@ class SweepConfig(BaseModel): class SweepCommand(ClientCommand): def run(self, sweep_name: str, num_trials: str) -> None: config = SweepConfig(sweep_name=sweep_name, num_trials=num_trials) - _ = self.invoke_handler(config=config) + response = self.invoke_handler(config=config) + print("Response Client", response) class FlowCommands(LightningFlow): @@ -33,6 +34,7 @@ def trigger_method(self, name: str): def sweep(self, config: SweepConfig): print(config) + return True def configure_commands(self): return [{"user_command": self.trigger_method}, {"sweep": SweepCommand(self.sweep)}] From bc9b2f58adbdcb727360c25534f675225e7528e0 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 14 Jul 2022 20:31:44 +0100 Subject: [PATCH 21/61] update --- src/lightning_app/core/api.py | 25 +++++++++++++------ src/lightning_app/core/app.py | 2 ++ src/lightning_app/core/queues.py | 5 ++++ src/lightning_app/runners/backends/backend.py | 1 + src/lightning_app/runners/multiprocess.py | 4 +-- tests/tests_app/utilities/test_commands.py | 2 +- 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index a94524645add8..4af9145e66c00 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -42,6 +42,7 @@ class SessionMiddleware: api_app_delta_queue: Queue = None api_commands_requests_queue: Queue = None api_commands_metadata_queue: Queue = None +api_commands_responses_queue: Queue = None template = {"ui": {}, "app": {}} templates = Jinja2Templates(directory=FRONTEND_DIR) @@ -174,12 +175,19 @@ async def post_command( affiliation = data.get("affiliation", None) if not affiliation: raise Exception("The provided affiliation is empty.") - data = await request.json() - # request_id = data["id"] - api_commands_requests_queue.put(data) - # response = api_commands_requests_queue.get() - # if request_id == response["response_id"]: - # return response + + async def fn(request: Request): + data = await request.json() + request_id = data["id"] + api_commands_requests_queue.put(data) + + response = api_commands_responses_queue.get() + if request_id == response["id"]: + return response["response"] + else: + raise Exception("This is a bug") + + return await asyncio.create_task(fn(request)) @fastapi_service.get("/api/v1/commands", response_class=JSONResponse) @@ -323,7 +331,7 @@ def start_server( api_publish_state_queue, api_delta_queue, commands_requests_queue, - commands_response_queue, + commands_responses_queue, commands_metadata_queue, has_started_queue: Optional[Queue] = None, host="127.0.0.1", @@ -335,10 +343,13 @@ def start_server( global api_app_delta_queue global global_app_state_store global api_commands_requests_queue + global api_commands_responses_queue global app_spec + app_spec = spec api_app_delta_queue = api_delta_queue api_commands_requests_queue = commands_requests_queue + api_commands_responses_queue = commands_responses_queue api_commands_metadata_queue = commands_metadata_queue if app_state_store is not None: diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 8dbd838de3351..7630897760b3c 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -75,6 +75,7 @@ def __init__( self.delta_queue: t.Optional[BaseQueue] = None self.readiness_queue: t.Optional[BaseQueue] = None self.commands_requests_queue: t.Optional[BaseQueue] = None + self.commands_responses_queue: t.Optional[BaseQueue] = None self.commands_metadata_queue: t.Optional[BaseQueue] = None self.api_publish_state_queue: t.Optional[BaseQueue] = None self.api_delta_queue: t.Optional[BaseQueue] = None @@ -379,6 +380,7 @@ def apply_commands(self): # 2.1: Evaluate the method associated to a specific command. # Validation is done on the CLI side. response = method(**command_query["command_arguments"]) + self.commands_responses_queue.put({"response": response, "id": command_query["id"]}) def run_once(self): """Method used to collect changes and run the root Flow once.""" diff --git a/src/lightning_app/core/queues.py b/src/lightning_app/core/queues.py index 640d369977d80..74f3d7f342a6d 100644 --- a/src/lightning_app/core/queues.py +++ b/src/lightning_app/core/queues.py @@ -37,6 +37,7 @@ ORCHESTRATOR_COPY_RESPONSE_CONSTANT = "ORCHESTRATOR_COPY_RESPONSE" WORK_QUEUE_CONSTANT = "WORK_QUEUE" COMMANDS_REQUESTS_QUEUE_CONSTANT = "COMMANDS_REQUESTS_QUEUE" +COMMANDS_RESPONSES_QUEUE_CONSTANT = "COMMANDS_RESPONSES_QUEUE" COMMANDS_METADATA_QUEUE_CONSTANT = "COMMANDS_METADATA_QUEUE" @@ -57,6 +58,10 @@ def get_commands_requests_queue(self, queue_id: Optional[str] = None) -> "BaseQu queue_name = f"{queue_id}_{COMMANDS_REQUESTS_QUEUE_CONSTANT}" if queue_id else COMMANDS_REQUESTS_QUEUE_CONSTANT return self._get_queue(queue_name) + def get_commands_responses_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": + queue_name = f"{queue_id}_{COMMANDS_REQUESTS_QUEUE_CONSTANT}" if queue_id else COMMANDS_REQUESTS_QUEUE_CONSTANT + return self._get_queue(queue_name) + def get_commands_metadata_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": queue_name = f"{queue_id}_{COMMANDS_METADATA_QUEUE_CONSTANT}" if queue_id else COMMANDS_METADATA_QUEUE_CONSTANT return self._get_queue(queue_name) diff --git a/src/lightning_app/runners/backends/backend.py b/src/lightning_app/runners/backends/backend.py index 643f15f0cf6d7..c370c7098b778 100644 --- a/src/lightning_app/runners/backends/backend.py +++ b/src/lightning_app/runners/backends/backend.py @@ -83,6 +83,7 @@ def _prepare_queues(self, app): app.delta_queue = self.queues.get_delta_queue(**kw) app.readiness_queue = self.queues.get_readiness_queue(**kw) app.commands_requests_queue = self.queues.get_commands_requests_queue(**kw) + app.commands_responses_queue = self.queues.get_commands_responses_queue(**kw) app.commands_metadata_queue = self.queues.get_commands_metadata_queue(**kw) app.error_queue = self.queues.get_error_queue(**kw) app.delta_queue = self.queues.get_delta_queue(**kw) diff --git a/src/lightning_app/runners/multiprocess.py b/src/lightning_app/runners/multiprocess.py index 61274f1b46967..92ec900d89c65 100644 --- a/src/lightning_app/runners/multiprocess.py +++ b/src/lightning_app/runners/multiprocess.py @@ -1,4 +1,3 @@ -import asyncio import multiprocessing import os from dataclasses import dataclass @@ -61,7 +60,6 @@ def dispatch(self, *args: Any, on_before_run: Optional[Callable] = None, **kwarg if self.start_server: self.app.should_publish_changes_to_api = True has_started_queue = self.backend.queues.get_has_server_started_queue() - self.app.commands_response_queue = asyncio.Queue() kwargs = dict( host=self.host, port=self.port, @@ -69,7 +67,7 @@ def dispatch(self, *args: Any, on_before_run: Optional[Callable] = None, **kwarg api_delta_queue=self.app.api_delta_queue, has_started_queue=has_started_queue, commands_requests_queue=self.app.commands_requests_queue, - commands_response_queue=self.app.commands_response_queue, + commands_responses_queue=self.app.commands_responses_queue, commands_metadata_queue=self.app.commands_metadata_queue, spec=extract_metadata_from_app(self.app), ) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index a892b0494ee7f..8d141065c7675 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -21,7 +21,7 @@ class SweepCommand(ClientCommand): def run(self, sweep_name: str, num_trials: str) -> None: config = SweepConfig(sweep_name=sweep_name, num_trials=num_trials) response = self.invoke_handler(config=config) - print("Response Client", response) + assert response is True class FlowCommands(LightningFlow): From 870b9aa03034619f8b93a0eadecaf8b5f592ebcf Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 14 Jul 2022 20:52:23 +0100 Subject: [PATCH 22/61] update --- tests/tests_app/core/test_lightning_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/tests_app/core/test_lightning_api.py b/tests/tests_app/core/test_lightning_api.py index 0173d45fcf9ce..4d228348e7340 100644 --- a/tests/tests_app/core/test_lightning_api.py +++ b/tests/tests_app/core/test_lightning_api.py @@ -192,6 +192,7 @@ def get(self, timeout: int = 0): change_state_queue = MockQueue("change_state_queue") has_started_queue = MockQueue("has_started_queue") commands_requests_queue = MockQueue("commands_requests_queue") + commands_responses_queue = MockQueue("commands_responses_queue") commands_metadata_queue = MockQueue("commands_metadata_queue") state = app.state_with_changes publish_state_queue.put(state) @@ -200,6 +201,7 @@ def get(self, timeout: int = 0): publish_state_queue, change_state_queue, commands_requests_queue, + commands_responses_queue, commands_metadata_queue, has_started_queue=has_started_queue, uvicorn_run=False, @@ -341,12 +343,14 @@ def test_start_server_started(): api_delta_queue = mp.Queue() has_started_queue = mp.Queue() commands_requests_queue = mp.Queue() + commands_responses_queue = mp.Queue() commands_metadata_queue = mp.Queue() kwargs = dict( api_publish_state_queue=api_publish_state_queue, api_delta_queue=api_delta_queue, has_started_queue=has_started_queue, commands_requests_queue=commands_requests_queue, + commands_responses_queue=commands_responses_queue, commands_metadata_queue=commands_metadata_queue, port=1111, ) @@ -368,6 +372,7 @@ def test_start_server_info_message(ui_refresher, uvicorn_run, caplog, monkeypatc api_delta_queue = MockQueue() has_started_queue = MockQueue() commands_requests_queue = MockQueue() + commands_responses_queue = MockQueue() commands_metadata_queue = MockQueue() kwargs = dict( host=host, @@ -376,6 +381,7 @@ def test_start_server_info_message(ui_refresher, uvicorn_run, caplog, monkeypatc api_delta_queue=api_delta_queue, has_started_queue=has_started_queue, commands_requests_queue=commands_requests_queue, + commands_responses_queue=commands_responses_queue, commands_metadata_queue=commands_metadata_queue, ) From c1de86bc9c635bf75f0293ff2d65729568935ded Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 15 Jul 2022 12:58:35 +0100 Subject: [PATCH 23/61] update --- .actions/delete_cloud_lightning_apps.py | 34 ------------------------- .actions/download_frontend.py | 5 ---- examples/app_commands/.lightning | 2 +- setup.py | 2 +- src/lightning_app/CHANGELOG.md | 2 +- 5 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 .actions/delete_cloud_lightning_apps.py delete mode 100644 .actions/download_frontend.py diff --git a/.actions/delete_cloud_lightning_apps.py b/.actions/delete_cloud_lightning_apps.py deleted file mode 100644 index f682847f1a608..0000000000000 --- a/.actions/delete_cloud_lightning_apps.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -from lightning_cloud.openapi.rest import ApiException - -from lightning_app.utilities.cloud import _get_project -from lightning_app.utilities.network import LightningClient - -client = LightningClient() - -try: - PR_NUMBER = int(os.getenv("PR_NUMBER", None)) -except (TypeError, ValueError): - # Failed when the PR is running master or 'PR_NUMBER' isn't defined. - PR_NUMBER = "" - -APP_NAME = os.getenv("TEST_APP_NAME", "") - -project = _get_project(client) -list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) - -print([lightningapp.name for lightningapp in list_lightningapps.lightningapps]) - -for lightningapp in list_lightningapps.lightningapps: - if PR_NUMBER and APP_NAME and not lightningapp.name.startswith(f"test-{PR_NUMBER}-{APP_NAME}-"): - continue - print(f"Deleting {lightningapp.name}") - try: - res = client.lightningapp_instance_service_delete_lightningapp_instance( - project_id=project.project_id, - id=lightningapp.id, - ) - assert res == {} - except ApiException as e: - print(f"Failed to delete {lightningapp.name}. Exception {e}") diff --git a/.actions/download_frontend.py b/.actions/download_frontend.py deleted file mode 100644 index 44d38865803b5..0000000000000 --- a/.actions/download_frontend.py +++ /dev/null @@ -1,5 +0,0 @@ -import lightning_app -from lightning_app.utilities.packaging.lightning_utils import download_frontend - -if __name__ == "__main__": - download_frontend(lightning_app._PROJECT_ROOT) diff --git a/examples/app_commands/.lightning b/examples/app_commands/.lightning index cebe537814b39..3efc0ce6284b0 100644 --- a/examples/app_commands/.lightning +++ b/examples/app_commands/.lightning @@ -1 +1 @@ -name: test_13 +name: app-commands diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index b881d12779874..9cef9efcdfd06 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). -## \[0.5.2\] - 2022-MM-DD +## [0.6.0] - 2022-MM-DD ### Added From 9d7eb188080d41a3bca1d275a2fc05bfde76452e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Jul 2022 12:02:32 +0000 Subject: [PATCH 24/61] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index 9cef9efcdfd06..49e5c60727cf6 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). -## [0.6.0] - 2022-MM-DD +## \[0.6.0\] - 2022-MM-DD ### Added From 3c3192f155c38111bddab9e27b4a934a692e2382 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 10:45:02 +0200 Subject: [PATCH 25/61] update --- src/lightning_app/core/app.py | 19 ++------ src/lightning_app/utilities/commands/base.py | 18 +++++++- tests/tests_app/utilities/test_commands.py | 46 +++++++++++++++++++- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 7630897760b3c..35693b9dee38d 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -3,7 +3,6 @@ import os import pickle import queue -import sys import threading import typing as t import warnings @@ -329,7 +328,7 @@ def maybe_apply_changes(self) -> bool: def apply_commands(self): """This method is used to apply remotely a collection of commands (methods) from the CLI to a running app.""" - from lightning_app.utilities.commands.base import ClientCommand + from lightning_app.utilities.commands.base import _command_to_method_and_metadata, ClientCommand if not is_overridden("configure_commands", self.root): return @@ -341,19 +340,9 @@ def apply_commands(self): for command_mapping in commands: for command_name, command in command_mapping.items(): is_command = isinstance(command, ClientCommand) - extra = {} + extras = {} if is_command: - params = inspect.signature(command.method).parameters - extra = { - "cls_path": inspect.getfile(command.__class__), - "cls_name": command.__class__.__name__, - "params": {p.name: str(p.annotation).split("'")[1].split(".")[-1] for p in params.values()}, - **command._to_dict(), - } - command.models = { - k: getattr(sys.modules[command.__module__], v) for k, v in extra["params"].items() - } - command = command.method + command, extras = _command_to_method_and_metadata(command) if command_name in command_names: raise Exception(f"The component name {command_name} has already been used. They need to be unique.") command_names.add(command_name) @@ -364,7 +353,7 @@ def apply_commands(self): "affiliation": command.__self__.name, "params": list(params.keys()), "is_command": is_command, - **extra, + **extras, } ) diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 5d8f934085a44..2f277cc74bf86 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -1,4 +1,5 @@ import errno +import inspect import os import os.path as osp import shutil @@ -50,7 +51,7 @@ def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], url: st self.models = models self.url = url - def run(self): + def run(self, **cli_kwargs) -> None: """Overrides with the logic to execute on the client side.""" def invoke_handler(self, **kwargs: Any) -> Dict[str, Any]: @@ -78,7 +79,6 @@ def __call__(self, **kwargs: Any) -> Any: def _download_command(command_metadata: Dict[str, Any]) -> Tuple[ClientCommand, Dict[str, BaseModel]]: config = _Config(**command_metadata) - print(config) if config.cls_path.startswith("s3://"): raise NotImplementedError() else: @@ -95,3 +95,17 @@ def _download_command(command_metadata: Dict[str, Any]) -> Tuple[ClientCommand, models = {k: getattr(mod, v) for k, v in config.params.items()} shutil.rmtree(tmpdir) return command, models + + +def _command_to_method_and_metadata(command: ClientCommand) -> Tuple[Callable, Dict[str, Any]]: + """Extract method and metadata from a ClientCommand.""" + params = inspect.signature(command.method).parameters + extra = { + "cls_path": inspect.getfile(command.__class__), + "cls_name": command.__class__.__name__, + "params": {p.name: str(p.annotation).split("'")[1].split(".")[-1] for p in params.values()}, + **command._to_dict(), + } + command.models = {k: getattr(sys.modules[command.__module__], v) for k, v in extra["params"].items()} + method = command.method + return method, extra diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 8d141065c7675..b7c28b1181243 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -1,5 +1,6 @@ from multiprocessing import Process from time import sleep +from unittest.mock import MagicMock from click.testing import CliRunner from pydantic import BaseModel @@ -8,7 +9,7 @@ from lightning_app import LightningApp from lightning_app.cli.lightning_cli import command from lightning_app.runners import MultiProcessRuntime -from lightning_app.utilities.commands.base import ClientCommand +from lightning_app.utilities.commands.base import _command_to_method_and_metadata, _download_command, ClientCommand from lightning_app.utilities.state import AppState @@ -40,6 +41,49 @@ def configure_commands(self): return [{"user_command": self.trigger_method}, {"sweep": SweepCommand(self.sweep)}] +class DummyConfig(BaseModel): + something: str + something_else: int + + +class DummyCommand(ClientCommand): + def run(self, something: str, something_else: int) -> None: + config = DummyConfig(something=something, something_else=something_else) + response = self.invoke_handler(config=config) + assert response == {"body": 0} + + +def run(config: DummyConfig): + assert isinstance(config, DummyCommand) + + +def test_client_commands(monkeypatch): + import requests + + resp = MagicMock() + resp.status_code = 200 + value = {"body": 0} + resp.json = MagicMock(return_value=value) + post = MagicMock() + post.return_value = resp + monkeypatch.setattr(requests, "post", post) + url = "http//" + kwargs = {"something": "1", "something_else": "1"} + command = DummyCommand(run) + _, command_metadata = _command_to_method_and_metadata(command) + command_metadata.update( + { + "command": "dummy", + "affiliation": "root", + "is_command": True, + "owner": "root", + } + ) + client_command, models = _download_command(command_metadata) + client_command._setup(metadata=command_metadata, models=models, url=url) + client_command.run(**kwargs) + + def target(): app = LightningApp(FlowCommands()) MultiProcessRuntime(app).dispatch() From 42fe030bf571e1abc59357f43c88ae7aff8f5fa4 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 19:07:01 +0200 Subject: [PATCH 26/61] update --- examples/app_commands/app.py | 13 +++- examples/app_commands/command.py | 12 +++ src/lightning_app/cli/lightning_cli.py | 11 ++- src/lightning_app/core/app.py | 4 + src/lightning_app/utilities/cli_helpers.py | 2 + .../utilities/commands/__init__.py | 3 + src/lightning_app/utilities/commands/base.py | 74 ++++++++++++++----- .../utilities/packaging/lightning_utils.py | 2 + tests/tests_app/utilities/test_commands.py | 28 +++++++ 9 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 examples/app_commands/command.py diff --git a/examples/app_commands/app.py b/examples/app_commands/app.py index 4f3de3c355522..10d31e131bf01 100644 --- a/examples/app_commands/app.py +++ b/examples/app_commands/app.py @@ -1,3 +1,5 @@ +from command import CustomCommand, CustomConfig + from lightning import LightningFlow from lightning_app.core.app import LightningApp @@ -20,11 +22,18 @@ def run(self): if len(self.names): print(self.names) - def trigger_method(self, name: str): + def trigger_without_client_command(self, name: str): self.names.append(name) + def trigger_with_client_command(self, config: CustomConfig): + self.names.append(config.name) + def configure_commands(self): - return [{"flow_trigger_command": self.trigger_method}] + self.child_flow.configure_commands() + commnands = [ + {"trigger_without_client_command": self.trigger_without_client_command}, + {"trigger_with_client_command": CustomCommand(self.trigger_with_client_command)}, + ] + return commnands + self.child_flow.configure_commands() app = LightningApp(FlowCommands()) diff --git a/examples/app_commands/command.py b/examples/app_commands/command.py new file mode 100644 index 0000000000000..d0faa561195a9 --- /dev/null +++ b/examples/app_commands/command.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel + +from lightning_app.utilities.commands import ClientCommand + + +class CustomConfig(BaseModel): + name: str + + +class CustomCommand(ClientCommand): + def run(self, name: str): + self.invoke_handler(config=CustomConfig(name=name)) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index a4915648cf4be..82fbf8bb89626 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -142,6 +142,7 @@ def command( """Execute a function in a running application from its name.""" from lightning_app.utilities.commands.base import _download_command + # 1: Collect the url and comments from the running application url, commands = _retrieve_application_url_and_available_commands(id) if url is None or commands is None: raise Exception("We couldn't find any matching running app.") @@ -153,9 +154,15 @@ def command( if command not in command_names: raise Exception(f"The provided command {command} isn't available in {command_names}") + # 2: Send the command from the user command_metadata = [c for c in commands if c["command"] == command][0] params = command_metadata["params"] + + # 3: Prepare the arguments provided by the users. + # TODO: Improve what is supported there. kwargs = {k.split("=")[0]: k.split("=")[1] for k in args} + + # 4: Execute commands if not command_metadata["is_command"]: for param in params: if param not in kwargs: @@ -169,8 +176,8 @@ def command( resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() else: - client_command, models = _download_command(command_metadata) - client_command._setup(metadata=command_metadata, models=models, url=url) + client_command, models = _download_command(command_metadata, id) + client_command._setup(metadata=command_metadata, models=models, app_url=url) client_command.run(**kwargs) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 35693b9dee38d..47838d372da78 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -17,6 +17,7 @@ from lightning_app.frontend import Frontend from lightning_app.storage.path import storage_root_dir from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef, is_overridden +from lightning_app.utilities.commands.base import _upload_command from lightning_app.utilities.component import _convert_paths_after_init from lightning_app.utilities.enum import AppStage from lightning_app.utilities.exceptions import CacheMissException, ExitAppException @@ -342,6 +343,7 @@ def apply_commands(self): is_command = isinstance(command, ClientCommand) extras = {} if is_command: + _upload_command(command_name, command) command, extras = _command_to_method_and_metadata(command) if command_name in command_names: raise Exception(f"The component name {command_name} has already been used. They need to be unique.") @@ -365,6 +367,8 @@ def apply_commands(self): if command_query: for command in commands: for command_name, method in command.items(): + logger.info(command_query) + logger.info(command_name) if command_query["command_name"] == command_name: # 2.1: Evaluate the method associated to a specific command. # Validation is done on the CLI side. diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index c8e8018c49119..45c5f9c89ed9f 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -86,6 +86,8 @@ def _retrieve_application_url_and_available_commands(app_id_or_name_or_url: Opti for lightningapp in list_lightningapps.lightningapps: if lightningapp.id == app_id_or_name_or_url or lightningapp.name == app_id_or_name_or_url: + if lightningapp.status.url == "": + raise Exception("The application is starting. Try in a few moments.") resp = requests.get(lightningapp.status.url + "/api/v1/commands") if resp.status_code != 200: raise Exception(f"The server didn't process the request properly. Found {resp.json()}") diff --git a/src/lightning_app/utilities/commands/__init__.py b/src/lightning_app/utilities/commands/__init__.py index e69de29bb2d1d..2ae6aba120168 100644 --- a/src/lightning_app/utilities/commands/__init__.py +++ b/src/lightning_app/utilities/commands/__init__.py @@ -0,0 +1,3 @@ +from lightning_app.utilities.commands.base import ClientCommand + +__all__ = ["ClientCommand"] diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 2f277cc74bf86..632023511ca72 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -1,5 +1,6 @@ import errno import inspect +import logging import os import os.path as osp import shutil @@ -13,7 +14,10 @@ import requests from pydantic import BaseModel -from lightning.app.utilities.state import headers_for +from lightning_app.utilities.cloud import _get_project +from lightning_app.utilities.network import LightningClient + +_logger = logging.getLogger(__name__) def makedirs(path: str): @@ -44,17 +48,19 @@ def __init__(self, method: Callable, requirements: Optional[List[str]] = None) - self.requirements = requirements self.metadata = None self.models = Optional[Dict[str, BaseModel]] - self.url = None + self.app_url = None - def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], url: str) -> None: + def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], app_url: str) -> None: self.metadata = metadata self.models = models - self.url = url + self.app_url = app_url def run(self, **cli_kwargs) -> None: """Overrides with the logic to execute on the client side.""" def invoke_handler(self, **kwargs: Any) -> Dict[str, Any]: + from lightning.app.utilities.state import headers_for + assert kwargs.keys() == self.models.keys() for k, v in kwargs.items(): assert isinstance(v, self.models[k]) @@ -64,7 +70,7 @@ def invoke_handler(self, **kwargs: Any) -> Dict[str, Any]: "affiliation": self.metadata["affiliation"], "id": str(uuid4()), } - resp = requests.post(self.url + "/api/v1/commands", json=json, headers=headers_for({})) + resp = requests.post(self.app_url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() return resp.json() @@ -77,24 +83,34 @@ def __call__(self, **kwargs: Any) -> Any: return self.method(**kwargs) -def _download_command(command_metadata: Dict[str, Any]) -> Tuple[ClientCommand, Dict[str, BaseModel]]: +def _download_command( + command_metadata: Dict[str, Any], app_id: Optional[str] +) -> Tuple[ClientCommand, Dict[str, BaseModel]]: config = _Config(**command_metadata) - if config.cls_path.startswith("s3://"): - raise NotImplementedError() + tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") + makedirs(tmpdir) + target_file = osp.join(tmpdir, f"{config.command}.py") + if app_id: + client = LightningClient() + project_id = _get_project(client).project_id + response = client.lightningapp_instance_service_list_lightningapp_instance_artifacts(project_id, app_id) + for artifact in response.artifacts: + if f"commands/{config.command}.py" == artifact.filename: + r = requests.get(artifact.url, allow_redirects=True) + with open(target_file, "wb") as f: + f.write(r.content) else: - tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") - makedirs(tmpdir) - cls_name = config.cls_name - target_file = osp.join(tmpdir, f"{config.command}.py") shutil.copy(config.cls_path, target_file) - spec = spec_from_file_location(config.cls_name, target_file) - mod = module_from_spec(spec) - sys.modules[cls_name] = mod - spec.loader.exec_module(mod) - command = getattr(mod, cls_name)(method=None, requirements=config.requirements) - models = {k: getattr(mod, v) for k, v in config.params.items()} - shutil.rmtree(tmpdir) - return command, models + + cls_name = config.cls_name + spec = spec_from_file_location(config.cls_name, target_file) + mod = module_from_spec(spec) + sys.modules[cls_name] = mod + spec.loader.exec_module(mod) + command = getattr(mod, cls_name)(method=None, requirements=config.requirements) + models = {k: getattr(mod, v) for k, v in config.params.items()} + shutil.rmtree(tmpdir) + return command, models def _command_to_method_and_metadata(command: ClientCommand) -> Tuple[Callable, Dict[str, Any]]: @@ -109,3 +125,21 @@ def _command_to_method_and_metadata(command: ClientCommand) -> Tuple[Callable, D command.models = {k: getattr(sys.modules[command.__module__], v) for k, v in extra["params"].items()} method = command.method return method, extra + + +def _upload_command(command_name: str, command: ClientCommand) -> Optional[str]: + from lightning_app.storage.path import _is_s3fs_available, filesystem, shared_storage_path + + filepath = f"commands/{command_name}.py" + remote_url = str(shared_storage_path() / "artifacts" / filepath) + fs = filesystem() + + if _is_s3fs_available() and not fs.exists(remote_url): + from s3fs import S3FileSystem + + if not isinstance(fs, S3FileSystem): + return + source_file = str(inspect.getfile(command.__class__)) + remote_url = str(shared_storage_path() / "artifacts" / filepath) + fs.put(source_file, remote_url) + return filepath diff --git a/src/lightning_app/utilities/packaging/lightning_utils.py b/src/lightning_app/utilities/packaging/lightning_utils.py index c6bcea035797f..b133eba48f250 100644 --- a/src/lightning_app/utilities/packaging/lightning_utils.py +++ b/src/lightning_app/utilities/packaging/lightning_utils.py @@ -113,6 +113,7 @@ def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable] # building and copying launcher wheel if installed in editable mode launcher_project_path = get_dist_path_if_editable_install("lightning_launcher") if launcher_project_path: + logger.info("Packaged Lightning Launcher with your application.") _prepare_wheel(launcher_project_path) tar_name = _copy_tar(launcher_project_path, root) tar_files.append(os.path.join(root, tar_name)) @@ -120,6 +121,7 @@ def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable] # building and copying lightning-cloud wheel if installed in editable mode lightning_cloud_project_path = get_dist_path_if_editable_install("lightning_cloud") if lightning_cloud_project_path: + logger.info("Packaged Lightning Cloud with your application.") _prepare_wheel(lightning_cloud_project_path) tar_name = _copy_tar(lightning_cloud_project_path, root) tar_files.append(os.path.join(root, tar_name)) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index b7c28b1181243..eceae285a772b 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -2,6 +2,7 @@ from time import sleep from unittest.mock import MagicMock +import pytest from click.testing import CliRunner from pydantic import BaseModel @@ -57,6 +58,33 @@ def run(config: DummyConfig): assert isinstance(config, DummyCommand) +def run_failure_0(name: str): + pass + + +def run_failure_1(name): + pass + + +class CustomModel(BaseModel): + pass + + +def run_failure_2(name: CustomModel): + pass + + +def test_command_to_method_and_metadata(): + with pytest.raises(Exception, match="The provided annotation for the argument name"): + _command_to_method_and_metadata(ClientCommand(run_failure_0)) + + with pytest.raises(Exception, match="Found _empty"): + _command_to_method_and_metadata(ClientCommand(run_failure_1)) + + with pytest.raises(Exception, match="The flow annotation needs to"): + _command_to_method_and_metadata(ClientCommand(run_failure_2)) + + def test_client_commands(monkeypatch): import requests From b796082cc64a5159cf6cb968a8b849c194cb077c Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 19:10:12 +0200 Subject: [PATCH 27/61] update --- tests/tests_app/core/test_lightning_flow.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/tests_app/core/test_lightning_flow.py b/tests/tests_app/core/test_lightning_flow.py index 74e9688898f45..26841e057621b 100644 --- a/tests/tests_app/core/test_lightning_flow.py +++ b/tests/tests_app/core/test_lightning_flow.py @@ -9,15 +9,15 @@ import pytest from deepdiff import DeepDiff, Delta -from lightning.app import LightningApp -from lightning.app.core.flow import LightningFlow -from lightning.app.core.work import LightningWork -from lightning.app.runners import MultiProcessRuntime, SingleProcessRuntime -from lightning.app.storage import Path -from lightning.app.storage.path import storage_root_dir -from lightning.app.testing.helpers import EmptyFlow, EmptyWork -from lightning.app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef -from lightning.app.utilities.exceptions import ExitAppException +from lightning_app import LightningApp +from lightning_app.core.flow import LightningFlow +from lightning_app.core.work import LightningWork +from lightning_app.runners import MultiProcessRuntime, SingleProcessRuntime +from lightning_app.storage import Path +from lightning_app.storage.path import storage_root_dir +from lightning_app.testing.helpers import EmptyFlow, EmptyWork +from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef +from lightning_app.utilities.exceptions import ExitAppException def test_empty_component(): @@ -543,7 +543,7 @@ def run(self): def test_flow_path_assignment(): - """Test that paths in the lit format lit:// get converted to a proper lightning.app.storage.Path object.""" + """Test that paths in the lit format lit:// get converted to a proper lightning_app.storage.Path object.""" class Flow(LightningFlow): def __init__(self): From 63a57f0c0dce6d61008b83e400fc501abe57bf9c Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 19:24:51 +0200 Subject: [PATCH 28/61] update --- src/lightning_app/core/api.py | 10 +++++----- src/lightning_app/core/app.py | 2 -- src/lightning_app/core/queues.py | 4 +++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index 4af9145e66c00..b34015697bff5 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -181,11 +181,11 @@ async def fn(request: Request): request_id = data["id"] api_commands_requests_queue.put(data) - response = api_commands_responses_queue.get() - if request_id == response["id"]: - return response["response"] - else: - raise Exception("This is a bug") + resp = api_commands_responses_queue.get() + if request_id == resp["id"]: + return resp["response"] + + raise Exception("This is a bug. It shouldn't happen.") return await asyncio.create_task(fn(request)) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 47838d372da78..ca8f0d7ccb1f4 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -367,8 +367,6 @@ def apply_commands(self): if command_query: for command in commands: for command_name, method in command.items(): - logger.info(command_query) - logger.info(command_name) if command_query["command_name"] == command_name: # 2.1: Evaluate the method associated to a specific command. # Validation is done on the CLI side. diff --git a/src/lightning_app/core/queues.py b/src/lightning_app/core/queues.py index 74f3d7f342a6d..efac8230047e0 100644 --- a/src/lightning_app/core/queues.py +++ b/src/lightning_app/core/queues.py @@ -59,7 +59,9 @@ def get_commands_requests_queue(self, queue_id: Optional[str] = None) -> "BaseQu return self._get_queue(queue_name) def get_commands_responses_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": - queue_name = f"{queue_id}_{COMMANDS_REQUESTS_QUEUE_CONSTANT}" if queue_id else COMMANDS_REQUESTS_QUEUE_CONSTANT + queue_name = ( + f"{queue_id}_{COMMANDS_RESPONSES_QUEUE_CONSTANT}" if queue_id else COMMANDS_RESPONSES_QUEUE_CONSTANT + ) return self._get_queue(queue_name) def get_commands_metadata_queue(self, queue_id: Optional[str] = None) -> "BaseQueue": From 4ed87593503742c5316bb1cd09478f2b6174fd00 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 19:50:06 +0200 Subject: [PATCH 29/61] update --- .gitignore | 1 + src/lightning_app/testing/testing.py | 6 +---- tests/tests_app_examples/test_commands.py | 29 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 tests/tests_app_examples/test_commands.py diff --git a/.gitignore b/.gitignore index ad4422b1a7ff7..19cba608e474f 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ celerybeat-schedule # dotenv .env +.env_stagging # virtualenv .venv diff --git a/src/lightning_app/testing/testing.py b/src/lightning_app/testing/testing.py index 7ae9bf6274e6c..7e52b231723ac 100644 --- a/src/lightning_app/testing/testing.py +++ b/src/lightning_app/testing/testing.py @@ -173,11 +173,7 @@ def run_app_in_cloud(app_folder: str, app_name: str = "app.py") -> Generator: # 5. Create chromium browser, auth to lightning_app.ai and yield the admin and view pages. with sync_playwright() as p: browser = p.chromium.launch(headless=bool(int(os.getenv("HEADLESS", "0")))) - payload = { - "apiKey": Config.api_key, - "username": Config.username, - "duration": "120000", - } + payload = {"apiKey": Config.api_key, "username": Config.username, "duration": "120000"} context = browser.new_context( # Eventually this will need to be deleted http_credentials=HttpCredentials({"username": os.getenv("LAI_USER"), "password": os.getenv("LAI_PASS")}), diff --git a/tests/tests_app_examples/test_commands.py b/tests/tests_app_examples/test_commands.py new file mode 100644 index 0000000000000..451f6b32026c3 --- /dev/null +++ b/tests/tests_app_examples/test_commands.py @@ -0,0 +1,29 @@ +import os +from subprocess import Popen +from time import sleep +from unittest import mock + +import pytest +from tests_app import _PROJECT_ROOT + +from lightning_app.testing.testing import run_app_in_cloud + + +@mock.patch.dict(os.environ, {"SKIP_LIGHTING_UTILITY_WHEELS_BUILD": "0"}) +@pytest.mark.cloud +def test_v0_app_example_cloud() -> None: + with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_commands")) as ( + admin_page, + _, + fetch_logs, + ): + app_id = admin_page.url.split("/")[-1] + cmd = f"lightning command trigger_with_client_command --args name=something --id {app_id}" + Popen(cmd, shell=True).wait() + + has_logs = False + while not has_logs: + for log in fetch_logs(): + if "['something']" in log: + has_logs = True + sleep(1) From e78e43b884b86d1b490accc4e945f1267bae3bc1 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 19:53:07 +0200 Subject: [PATCH 30/61] update --- .github/workflows/ci-app_cloud_e2e_test.yml | 1 + .gitignore | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ci-app_cloud_e2e_test.yml b/.github/workflows/ci-app_cloud_e2e_test.yml index 341c2dc20e83d..d0e035ec92337 100644 --- a/.github/workflows/ci-app_cloud_e2e_test.yml +++ b/.github/workflows/ci-app_cloud_e2e_test.yml @@ -54,6 +54,7 @@ jobs: - custom_work_dependencies - drive - payload + - commands timeout-minutes: 35 steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 19cba608e474f..7040a912974e1 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ tags .tags src/lightning_app/ui/* *examples/template_react_ui* +hars* +artifacts/* From 0f8b11fa2fd3da8f0d48ddb5b20d1278c45476ed Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 20:00:24 +0200 Subject: [PATCH 31/61] update --- src/lightning_app/core/flow.py | 4 ++-- tests/tests_app_examples/test_commands.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index c4295430af797..db8755b688217 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -629,10 +629,10 @@ def configure_commands(self): .. code-block:: bash - lightning run app app.py command + lightning run app app.py .. code-block:: bash - lightning run command add_name --args name=my_own_name + lightning command add_name --args name=my_own_name """ raise NotImplementedError diff --git a/tests/tests_app_examples/test_commands.py b/tests/tests_app_examples/test_commands.py index 451f6b32026c3..a30cdf1a4bb3f 100644 --- a/tests/tests_app_examples/test_commands.py +++ b/tests/tests_app_examples/test_commands.py @@ -11,7 +11,7 @@ @mock.patch.dict(os.environ, {"SKIP_LIGHTING_UTILITY_WHEELS_BUILD": "0"}) @pytest.mark.cloud -def test_v0_app_example_cloud() -> None: +def test_commands_example_cloud() -> None: with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_commands")) as ( admin_page, _, From 119b3120264868abde7bcd87f36920d571e52e62 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 20:47:18 +0200 Subject: [PATCH 32/61] update --- MANIFEST.in | 3 ++ setup.py | 2 +- src/lightning_app/utilities/cli_helpers.py | 4 +-- src/lightning_app/utilities/commands/base.py | 35 +++++++++++++++++--- tests/tests_app/utilities/test_commands.py | 24 ++++++++++---- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..9014f289aa9b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,6 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src/lightning_app/ui * diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index 45c5f9c89ed9f..f3300deafa71d 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -65,12 +65,12 @@ def _retrieve_application_url_and_available_commands(app_id_or_name_or_url: Opti if app_id_or_name_or_url is None: try: - url = f"http://127.0.0.1:{APP_SERVER_PORT}" + url = f"http://localhost:{APP_SERVER_PORT}" resp = requests.get(f"{url}/api/v1/commands") if resp.status_code != 200: raise Exception(f"The server didn't process the request properly. Found {resp.json()}") return url, resp.json() - except ConnectionError: + except requests.exceptions.ConnectionError: failed_locally = True # 3: If an identified was provided or the local evaluation has failed, evaluate the cloud. diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 632023511ca72..5c6e7ca8bc9fd 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -113,18 +113,43 @@ def _download_command( return command, models +def _to_annotation(anno: str) -> str: + anno = anno.split("'")[1] + if "." in anno: + return anno.split(".")[-1] + return anno + + def _command_to_method_and_metadata(command: ClientCommand) -> Tuple[Callable, Dict[str, Any]]: - """Extract method and metadata from a ClientCommand.""" + """Extract method and its metadata from a ClientCommand.""" params = inspect.signature(command.method).parameters - extra = { + command_metadata = { "cls_path": inspect.getfile(command.__class__), "cls_name": command.__class__.__name__, - "params": {p.name: str(p.annotation).split("'")[1].split(".")[-1] for p in params.values()}, + "params": {p.name: _to_annotation(str(p.annotation)) for p in params.values()}, **command._to_dict(), } - command.models = {k: getattr(sys.modules[command.__module__], v) for k, v in extra["params"].items()} method = command.method - return method, extra + command.models = {} + for k, v in command_metadata["params"].items(): + if v == "_empty": + raise Exception( + f"Please, annotate your method {method} with pydantic BaseModel. Refer to the documentation." + ) + config = getattr(sys.modules[command.__module__], v, None) + if config is None: + config = getattr(sys.modules[method.__module__], v, None) + if config: + raise Exception( + f"The provided annotation for the argument {k} should in the file " + f"{inspect.getfile(command.__class__)}, not {inspect.getfile(command.method)}." + ) + if not issubclass(config, BaseModel): + raise Exception( + f"The provided annotation for the argument {k} shouldn't an instance of pydantic BaseModel." + ) + command.models[k] = config + return method, command_metadata def _upload_command(command_name: str, command: ClientCommand) -> Optional[str]: diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index eceae285a772b..161f179973431 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -3,12 +3,14 @@ from unittest.mock import MagicMock import pytest +import requests from click.testing import CliRunner from pydantic import BaseModel from lightning import LightningFlow from lightning_app import LightningApp from lightning_app.cli.lightning_cli import command +from lightning_app.core.constants import APP_SERVER_PORT from lightning_app.runners import MultiProcessRuntime from lightning_app.utilities.commands.base import _command_to_method_and_metadata, _download_command, ClientCommand from lightning_app.utilities.state import AppState @@ -78,10 +80,10 @@ def test_command_to_method_and_metadata(): with pytest.raises(Exception, match="The provided annotation for the argument name"): _command_to_method_and_metadata(ClientCommand(run_failure_0)) - with pytest.raises(Exception, match="Found _empty"): + with pytest.raises(Exception, match="annotate your method"): _command_to_method_and_metadata(ClientCommand(run_failure_1)) - with pytest.raises(Exception, match="The flow annotation needs to"): + with pytest.raises(Exception, match="lightning_app/utilities/commands/base.py"): _command_to_method_and_metadata(ClientCommand(run_failure_2)) @@ -107,8 +109,8 @@ def test_client_commands(monkeypatch): "owner": "root", } ) - client_command, models = _download_command(command_metadata) - client_command._setup(metadata=command_metadata, models=models, url=url) + client_command, models = _download_command(command_metadata, None) + client_command._setup(metadata=command_metadata, models=models, app_url=url) client_command.run(**kwargs) @@ -120,15 +122,24 @@ def target(): def test_configure_commands(): process = Process(target=target) process.start() - sleep(5) + time_left = 15 + while time_left > 0: + try: + requests.get(f"http://localhost:{APP_SERVER_PORT}/healthz") + break + except requests.exceptions.ConnectionError: + sleep(0.1) + time_left -= 0.1 + + sleep(0.5) runner = CliRunner() result = runner.invoke( command, ["user_command", "--args", "name=something"], catch_exceptions=False, ) - sleep(2) assert result.exit_code == 0 + sleep(0.5) state = AppState() state._request_state() assert state.names == ["something"] @@ -138,5 +149,4 @@ def test_configure_commands(): ["sweep", "--args", "sweep_name=my_name", "--args", "num_trials=num_trials"], catch_exceptions=False, ) - sleep(2) assert result.exit_code == 0 From 88e6c133422444cd1733a71a451879d1a1266f18 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 18 Jul 2022 21:03:12 +0200 Subject: [PATCH 33/61] update --- src/lightning_app/utilities/commands/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 5c6e7ca8bc9fd..42a1d848efd2c 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -144,7 +144,7 @@ def _command_to_method_and_metadata(command: ClientCommand) -> Tuple[Callable, D f"The provided annotation for the argument {k} should in the file " f"{inspect.getfile(command.__class__)}, not {inspect.getfile(command.method)}." ) - if not issubclass(config, BaseModel): + if config is None or not issubclass(config, BaseModel): raise Exception( f"The provided annotation for the argument {k} shouldn't an instance of pydantic BaseModel." ) From aa0b60bdec81825e37767fe9d22e4bcb43de732c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 19 Jul 2022 01:07:24 +0200 Subject: [PATCH 34/61] Apply suggestions from code review Co-authored-by: Luca Antiga --- src/lightning_app/core/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index ca8f0d7ccb1f4..a4012cc7046db 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -346,7 +346,7 @@ def apply_commands(self): _upload_command(command_name, command) command, extras = _command_to_method_and_metadata(command) if command_name in command_names: - raise Exception(f"The component name {command_name} has already been used. They need to be unique.") + raise Exception(f"The command name {command_name} has already been used. They need to be unique.") command_names.add(command_name) params = inspect.signature(command).parameters commands_metadata.append( From 3e4fa95106eb0d6128ebee45ccdb1d7edf80c9d7 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Wed, 20 Jul 2022 09:46:52 +0200 Subject: [PATCH 35/61] update --- MANIFEST.in | 3 - examples/app_commands/app.py | 2 +- setup.py | 2 +- src/lightning_app/cli/lightning_cli.py | 5 +- src/lightning_app/core/api.py | 2 +- src/lightning_app/core/app.py | 56 ++---------------- src/lightning_app/core/flow.py | 33 +++++------ src/lightning_app/utilities/commands/base.py | 61 +++++++++++++++++++- tests/tests_app/utilities/test_commands.py | 2 +- 9 files changed, 86 insertions(+), 80 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9014f289aa9b5..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,3 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src/lightning_app/ui * diff --git a/examples/app_commands/app.py b/examples/app_commands/app.py index 10d31e131bf01..20e027c779289 100644 --- a/examples/app_commands/app.py +++ b/examples/app_commands/app.py @@ -6,7 +6,7 @@ class ChildFlow(LightningFlow): def trigger_method(self, name: str): - print(name) + print(f"Hello {name}") def configure_commands(self): return [{"nested_trigger_command": self.trigger_method}] diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 8ebc4e70430bb..30addead6810d 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -149,6 +149,9 @@ def command( id: Optional[str] = None, ): """Execute a function in a running application from its name.""" + + logger.warn("Lightning Commands are a beta feature and APIs aren't stable yet.") + from lightning_app.utilities.commands.base import _download_command # 1: Collect the url and comments from the running application @@ -172,7 +175,7 @@ def command( kwargs = {k.split("=")[0]: k.split("=")[1] for k in args} # 4: Execute commands - if not command_metadata["is_command"]: + if not command_metadata["is_client_command"]: for param in params: if param not in kwargs: raise Exception(f"The argument --args {param}=X hasn't been provided.") diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index b34015697bff5..11500baf443fe 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -162,7 +162,7 @@ async def get_spec( @fastapi_service.post("/api/v1/commands", response_class=JSONResponse) -async def post_command( +async def run_remote_command( request: Request, ) -> None: data = await request.json() diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index ca8f0d7ccb1f4..5686a5abe4db5 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -1,4 +1,3 @@ -import inspect import logging import os import pickle @@ -16,8 +15,8 @@ from lightning_app.core.queues import BaseQueue, SingleProcessQueue from lightning_app.frontend import Frontend from lightning_app.storage.path import storage_root_dir -from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef, is_overridden -from lightning_app.utilities.commands.base import _upload_command +from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef +from lightning_app.utilities.commands.base import _populate_commands_endpoint, _process_command_requests from lightning_app.utilities.component import _convert_paths_after_init from lightning_app.utilities.enum import AppStage from lightning_app.utilities.exceptions import CacheMissException, ExitAppException @@ -326,53 +325,6 @@ def maybe_apply_changes(self) -> bool: self.set_state(state) self._has_updated = True - def apply_commands(self): - """This method is used to apply remotely a collection of commands (methods) from the CLI to a running - app.""" - from lightning_app.utilities.commands.base import _command_to_method_and_metadata, ClientCommand - - if not is_overridden("configure_commands", self.root): - return - - # 1: Populate commands metadata - commands = self.root.configure_commands() - commands_metadata = [] - command_names = set() - for command_mapping in commands: - for command_name, command in command_mapping.items(): - is_command = isinstance(command, ClientCommand) - extras = {} - if is_command: - _upload_command(command_name, command) - command, extras = _command_to_method_and_metadata(command) - if command_name in command_names: - raise Exception(f"The component name {command_name} has already been used. They need to be unique.") - command_names.add(command_name) - params = inspect.signature(command).parameters - commands_metadata.append( - { - "command": command_name, - "affiliation": command.__self__.name, - "params": list(params.keys()), - "is_command": is_command, - **extras, - } - ) - - # 1.2: Pass the collected commands through the queue to the Rest API. - self.commands_metadata_queue.put(commands_metadata) - - # 2: Collect requests metadata - command_query = self.get_state_changed_from_queue(self.commands_requests_queue) - if command_query: - for command in commands: - for command_name, method in command.items(): - if command_query["command_name"] == command_name: - # 2.1: Evaluate the method associated to a specific command. - # Validation is done on the CLI side. - response = method(**command_query["command_arguments"]) - self.commands_responses_queue.put({"response": response, "id": command_query["id"]}) - def run_once(self): """Method used to collect changes and run the root Flow once.""" done = False @@ -397,7 +349,7 @@ def run_once(self): elif self.stage == AppStage.RESTARTING: return self._apply_restarting() - self.apply_commands() + _process_command_requests(self) try: self.check_error_queue() @@ -451,6 +403,8 @@ def _run(self) -> bool: self._reset_run_time_monitor() + _populate_commands_endpoint(self) + while not done: done = self.run_once() diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index db8755b688217..d24033841d590 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -79,19 +79,15 @@ def __init__(self): .. doctest:: >>> from lightning import LightningFlow - ... >>> class RootFlow(LightningFlow): - ... ... def __init__(self): ... super().__init__() ... self.counter = 0 - ... ... def run(self): ... self.counter += 1 ... >>> flow = RootFlow() >>> flow.run() - ... >>> assert flow.counter == 1 >>> assert flow.state["vars"]["counter"] == 1 """ @@ -356,11 +352,11 @@ def schedule( from lightning_app import LightningFlow - class Flow(LightningFlow): + class Flow(LightningFlow): def run(self): if self.schedule("hourly"): - # run some code once every hour. + print("run some code every hour") Arguments: cron_pattern: The cron pattern to provide. Learn more at https://crontab.guru/. @@ -375,8 +371,8 @@ def run(self): from lightning_app import LightningFlow from lightning_app.structures import List - class SchedulerDAG(LightningFlow): + class SchedulerDAG(LightningFlow): def __init__(self): super().__init__() self.dags = List() @@ -487,8 +483,10 @@ def configure_layout(self) -> Union[Dict[str, Any], List[Dict[str, Any]], Fronte from lightning_app.frontend import StaticWebFrontend + class Flow(LightningFlow): ... + def configure_layout(self): return StaticWebFrontend("path/to/folder/to/serve") @@ -498,26 +496,28 @@ def configure_layout(self): from lightning_app.frontend import StaticWebFrontend + class Flow(LightningFlow): ... + def configure_layout(self): return StreamlitFrontend(render_fn=my_streamlit_ui) + def my_streamlit_ui(state): # add your streamlit code here! + import streamlit as st + **Example:** Arrange the UI of my children in tabs (default UI by Lightning). .. code-block:: python class Flow(LightningFlow): - ... def configure_layout(self): return [ dict(name="First Tab", content=self.child0), dict(name="Second Tab", content=self.child1), - ... - # You can include direct URLs too dict(name="Lightning", content="https://lightning.ai"), ] @@ -612,18 +612,15 @@ def configure_commands(self): .. code-block:: python class Flow(LightningFlow): - ... def __init__(self): super().__init__() self.names = [] - def handle_name_request(name: str) - self.names.append(name) - def configure_commands(self): - return [ - {"add_name": self.handle_name_request} - ] + return {"my_command_name": self.my_remote_method} + + def my_remote_method(self, name): + self.names.append(name) Once the app is running with the following command: @@ -633,6 +630,6 @@ def configure_commands(self): .. code-block:: bash - lightning command add_name --args name=my_own_name + lightning command my_command_name --args name=my_own_name """ raise NotImplementedError diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 42a1d848efd2c..49c9fb1c8ef3b 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -14,6 +14,7 @@ import requests from pydantic import BaseModel +from lightning_app.utilities.app_helpers import is_overridden from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.network import LightningClient @@ -29,11 +30,11 @@ def makedirs(path: str): raise e -class _Config(BaseModel): +class _ClientCommandConfig(BaseModel): command: str affiliation: str params: Dict[str, str] - is_command: bool + is_client_command: bool cls_path: str cls_name: str owner: str @@ -86,7 +87,9 @@ def __call__(self, **kwargs: Any) -> Any: def _download_command( command_metadata: Dict[str, Any], app_id: Optional[str] ) -> Tuple[ClientCommand, Dict[str, BaseModel]]: - config = _Config(**command_metadata) + # TODO: This is a skateboard implementation and the final version will rely on versioned + # immutable commands for security concerns + config = _ClientCommandConfig(**command_metadata) tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") makedirs(tmpdir) target_file = osp.join(tmpdir, f"{config.command}.py") @@ -168,3 +171,55 @@ def _upload_command(command_name: str, command: ClientCommand) -> Optional[str]: remote_url = str(shared_storage_path() / "artifacts" / filepath) fs.put(source_file, remote_url) return filepath + + +def _populate_commands_endpoint(app): + if not is_overridden("configure_commands", app.root): + return + + # 1: Populate commands metadata + commands = app.root.configure_commands() + commands_metadata = [] + command_names = set() + for command_mapping in commands: + for command_name, command in command_mapping.items(): + is_client_command = isinstance(command, ClientCommand) + extras = {} + if is_client_command: + _upload_command(command_name, command) + command, extras = _command_to_method_and_metadata(command) + if command_name in command_names: + raise Exception(f"The component name {command_name} has already been used. They need to be unique.") + command_names.add(command_name) + params = inspect.signature(command).parameters + commands_metadata.append( + { + "command": command_name, + "affiliation": command.__self__.name, + "params": list(params.keys()), + "is_client_command": is_client_command, + **extras, + } + ) + + # 1.2: Pass the collected commands through the queue to the Rest API. + app.commands_metadata_queue.put(commands_metadata) + + +def _process_command_requests(app): + if not is_overridden("configure_commands", app.root): + return + + # 1: Populate commands metadata + commands = app.root.configure_commands() + + # 2: Collect requests metadata + command_query = app.get_state_changed_from_queue(app.commands_requests_queue) + if command_query: + for command in commands: + for command_name, method in command.items(): + if command_query["command_name"] == command_name: + # 2.1: Evaluate the method associated to a specific command. + # Validation is done on the CLI side. + response = method(**command_query["command_arguments"]) + app.commands_responses_queue.put({"response": response, "id": command_query["id"]}) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 161f179973431..5e217d2b30953 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -105,7 +105,7 @@ def test_client_commands(monkeypatch): { "command": "dummy", "affiliation": "root", - "is_command": True, + "is_client_command": True, "owner": "root", } ) From ddad86c4ebb879c272f119df21bddefed789432e Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Wed, 20 Jul 2022 14:56:42 +0200 Subject: [PATCH 36/61] update --- src/lightning_app/components/python/tracer.py | 4 ++-- src/lightning_app/core/app.py | 1 + src/lightning_app/source_code/uploader.py | 22 +++++++++++-------- src/lightning_app/utilities/commands/base.py | 20 +++++++++++++---- src/lightning_app/utilities/proxies.py | 4 ++++ 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/lightning_app/components/python/tracer.py b/src/lightning_app/components/python/tracer.py index 5605eee6b6d47..df18d296868d6 100644 --- a/src/lightning_app/components/python/tracer.py +++ b/src/lightning_app/components/python/tracer.py @@ -93,8 +93,6 @@ def __init__( :language: python """ super().__init__(**kwargs) - if not os.path.exists(script_path): - raise FileNotFoundError(f"The provided `script_path` {script_path}` wasn't found.") self.script_path = str(script_path) if isinstance(script_args, str): script_args = script_args.split(" ") @@ -105,6 +103,8 @@ def __init__( setattr(self, name, None) def run(self, **kwargs): + if not os.path.exists(self.script_path): + raise FileNotFoundError(f"The provided `script_path` {self.script_path}` wasn't found.") kwargs = {k: v.value if isinstance(v, Payload) else v for k, v in kwargs.items()} init_globals = globals() init_globals.update(kwargs) diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 5686a5abe4db5..6599b53efcb95 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -85,6 +85,7 @@ def __init__( self.copy_response_queues: t.Optional[t.Dict[str, BaseQueue]] = None self.caller_queues: t.Optional[t.Dict[str, BaseQueue]] = None self.work_queues: t.Optional[t.Dict[str, BaseQueue]] = None + self.commands: t.Optional[t.List] = None self.should_publish_changes_to_api = False self.component_affiliation = None diff --git a/src/lightning_app/source_code/uploader.py b/src/lightning_app/source_code/uploader.py index b3a77bc6334bc..7f3413d1e29b0 100644 --- a/src/lightning_app/source_code/uploader.py +++ b/src/lightning_app/source_code/uploader.py @@ -40,15 +40,15 @@ def __init__(self, presigned_url: str, source_file: str, total_size: int, name: self.name = name @staticmethod - def upload_s3_data(url: str, data: bytes, retries: int, disconnect_retry_wait_seconds: int) -> str: - """Send data to s3 url. + def upload_data(url: str, data, retries: int, disconnect_retry_wait_seconds: int) -> str: + """Send data to url. Parameters ---------- url: str - S3 url string to send data to + url string to send data to data: bytes - Bytes of data to send to S3 + Bytes of data to send to url retries: int Amount of retries disconnect_retry_wait_seconds: int @@ -65,10 +65,14 @@ def upload_s3_data(url: str, data: bytes, retries: int, disconnect_retry_wait_se retries = Retry(total=10) with requests.Session() as s: s.mount("https://", HTTPAdapter(max_retries=retries)) - response = s.put(url, data=data) - if "ETag" not in response.headers: - raise ValueError(f"Unexpected response from S3, response: {response.content}") - return response.headers["ETag"] + if url.startswith("s3://"): + resp = s.put(url, data=data) + if "ETag" not in resp.headers: + raise ValueError(f"Unexpected response from S3, response: {resp.content}") + return resp.headers["ETag"] + else: + resp = s.put(url, files={"file": data}) + return str(resp.status_code) except BrokenPipeError: time.sleep(disconnect_retry_wait_seconds) disconnect_retries -= 1 @@ -82,7 +86,7 @@ def upload(self) -> None: try: with open(self.source_file, "rb") as f: data = f.read() - self.upload_s3_data(self.presigned_url, data, self.retries, self.disconnect_retry_wait_seconds) + self.upload_data(self.presigned_url, data, self.retries, self.disconnect_retry_wait_seconds) self.progress.update(task_id, advance=len(data)) finally: self.progress.stop() diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 49c9fb1c8ef3b..1adf7976c586e 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -17,6 +17,7 @@ from lightning_app.utilities.app_helpers import is_overridden from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.network import LightningClient +from lightning_app.utilities.state import AppState _logger = logging.getLogger(__name__) @@ -48,14 +49,22 @@ def __init__(self, method: Callable, requirements: Optional[List[str]] = None) - self.owner = flow.name if flow else None self.requirements = requirements self.metadata = None - self.models = Optional[Dict[str, BaseModel]] + self.models: Optional[Dict[str, BaseModel]] = None self.app_url = None + self._state = None def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], app_url: str) -> None: self.metadata = metadata self.models = models self.app_url = app_url + @property + def state(self): + if self._state is None: + self._state = AppState() + self._state._request_state() + return self._state + def run(self, **cli_kwargs) -> None: """Overrides with the logic to execute on the client side.""" @@ -80,8 +89,10 @@ def _to_dict(self): def __call__(self, **kwargs: Any) -> Any: assert self.models - kwargs = {k: self.models[k].parse_raw(v) for k, v in kwargs.items()} - return self.method(**kwargs) + input = {} + for k, v in kwargs.items(): + input[k] = self.models[k].parse_raw(v) + return self.method(**input) def _download_command( @@ -204,6 +215,7 @@ def _populate_commands_endpoint(app): # 1.2: Pass the collected commands through the queue to the Rest API. app.commands_metadata_queue.put(commands_metadata) + app.commands = commands def _process_command_requests(app): @@ -211,7 +223,7 @@ def _process_command_requests(app): return # 1: Populate commands metadata - commands = app.root.configure_commands() + commands = app.commands # 2: Collect requests metadata command_query = app.get_state_changed_from_queue(app.commands_requests_queue) diff --git a/src/lightning_app/utilities/proxies.py b/src/lightning_app/utilities/proxies.py index ead681bff7788..c33e41bb70203 100644 --- a/src/lightning_app/utilities/proxies.py +++ b/src/lightning_app/utilities/proxies.py @@ -5,6 +5,7 @@ import sys import threading import time +import traceback import warnings from copy import deepcopy from dataclasses import dataclass @@ -398,6 +399,9 @@ def run_once(self): ) self.delta_queue.put(ComponentDelta(id=self.work_name, delta=Delta(DeepDiff(state, self.work.state)))) self.work.on_exception(e) + print("########## CAPTURED EXCEPTION ###########") + print(traceback.print_exc()) + print("########## CAPTURED EXCEPTION ###########") return # 14. Copy all artifacts to the shared storage so other Works can access them while this Work gets scaled down From 7882a9e6ce85007914619115597ca161c61ecdc8 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 21 Jul 2022 13:24:51 +0200 Subject: [PATCH 37/61] update --- src/lightning_app/cli/lightning_cli.py | 69 ++++++++++---------- src/lightning_app/utilities/commands/base.py | 10 +-- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 30addead6810d..b9979a2735bc6 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -1,13 +1,13 @@ import logging import os from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Tuple, Union, Optional from uuid import uuid4 - +import sys import click import requests from requests.exceptions import ConnectionError - +from argparse import ArgumentParser from lightning_app import __version__ as ver from lightning_app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init from lightning_app.core.constants import get_lightning_cloud_url, LOCAL_LAUNCH_ADMIN_VIEW @@ -32,14 +32,22 @@ def get_app_url(runtime_type: RuntimeType, *args) -> str: return "http://127.0.0.1:7501/admin" if LOCAL_LAUNCH_ADMIN_VIEW else "http://127.0.0.1:7501/view" +def main(): + if len(sys.argv) == 1: + _main() + elif sys.argv[1] in _main.commands.keys(): + _main() + else: + app_command() + @click.group() @click.version_option(ver) -def main(): +def _main(): register_all_external_components() pass -@main.command() +@_main.command() def login(): """Log in to your Lightning.ai account.""" auth = Auth() @@ -52,7 +60,7 @@ def login(): exit(1) -@main.command() +@_main.command() def logout(): """Log out of your Lightning.ai account.""" Auth().clear() @@ -99,7 +107,7 @@ def on_before_run(*args): click.echo("Application is ready in the cloud") -@main.group() +@_main.group() def run(): """Run your application.""" @@ -131,37 +139,26 @@ def run_app( _run_app(file, cloud, without_server, no_cache, name, blocking, open_ui, env) -@main.command("command") -@click.argument("command", type=str, default="") -@click.option( - "--args", - type=str, - default=[], - multiple=True, - help="Arguments to be passed to the method executed in the running app.", -) -@click.option( - "--id", help="Unique identifier for the application. It can be its ID, its url or its name.", default=None, type=str -) -def command( - command: str, - args: List[str], - id: Optional[str] = None, -): +def app_command(): """Execute a function in a running application from its name.""" + from lightning_app.utilities.commands.base import _download_command logger.warn("Lightning Commands are a beta feature and APIs aren't stable yet.") - from lightning_app.utilities.commands.base import _download_command + parser = ArgumentParser() + parser.add_argument("--app_id", default=None, type=Optional[str], help="Optional argument to identify an application.") + hparams, _ = parser.parse_known_args() # 1: Collect the url and comments from the running application - url, commands = _retrieve_application_url_and_available_commands(id) + url, commands = _retrieve_application_url_and_available_commands(hparams.app_id) if url is None or commands is None: raise Exception("We couldn't find any matching running app.") if not commands: raise Exception("This application doesn't expose any commands yet.") + command = sys.argv[1] + command_names = [c["command"] for c in commands] if command not in command_names: raise Exception(f"The provided command {command} isn't available in {command_names}") @@ -172,10 +169,10 @@ def command( # 3: Prepare the arguments provided by the users. # TODO: Improve what is supported there. - kwargs = {k.split("=")[0]: k.split("=")[1] for k in args} # 4: Execute commands if not command_metadata["is_client_command"]: + kwargs = {k.split("=")[0]: k.split("=")[1] for k in sys.argv[2:]} for param in params: if param not in kwargs: raise Exception(f"The argument --args {param}=X hasn't been provided.") @@ -188,36 +185,36 @@ def command( resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() else: - client_command, models = _download_command(command_metadata, id) + client_command, models = _download_command(command_metadata, hparams.app_id) client_command._setup(metadata=command_metadata, models=models, app_url=url) - client_command.run(**kwargs) + client_command.run() -@main.group(hidden=True) +@_main.group(hidden=True) def fork(): """Fork an application.""" pass -@main.group(hidden=True) +@_main.group(hidden=True) def stop(): """Stop your application.""" pass -@main.group(hidden=True) +@_main.group(hidden=True) def delete(): """Delete an application.""" pass -@main.group(name="list", hidden=True) +@_main.group(name="list", hidden=True) def get_list(): """List your applications.""" pass -@main.group() +@_main.group() def install(): """Install Lightning apps and components.""" @@ -275,7 +272,7 @@ def install_component(name, yes, version): cmd_install.gallery_component(name, yes, version) -@main.group() +@_main.group() def init(): """Init a Lightning app and component.""" @@ -340,4 +337,4 @@ def _prepare_file(file: str) -> str: if exists: return file - raise FileNotFoundError(f"The provided file {file} hasn't been found.") + raise FileNotFoundError(f"The provided file {file} hasn't been found.") \ No newline at end of file diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 1adf7976c586e..9d68b58bfda59 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -101,9 +101,9 @@ def _download_command( # TODO: This is a skateboard implementation and the final version will rely on versioned # immutable commands for security concerns config = _ClientCommandConfig(**command_metadata) - tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") - makedirs(tmpdir) - target_file = osp.join(tmpdir, f"{config.command}.py") + # tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") + #makedirs(tmpdir) + target_file = osp.join(".", f"{config.command}.py") if app_id: client = LightningClient() project_id = _get_project(client).project_id @@ -116,6 +116,8 @@ def _download_command( else: shutil.copy(config.cls_path, target_file) + breakpoint() + cls_name = config.cls_name spec = spec_from_file_location(config.cls_name, target_file) mod = module_from_spec(spec) @@ -123,7 +125,7 @@ def _download_command( spec.loader.exec_module(mod) command = getattr(mod, cls_name)(method=None, requirements=config.requirements) models = {k: getattr(mod, v) for k, v in config.params.items()} - shutil.rmtree(tmpdir) + #shutil.rmtree(tmpdir) return command, models From aa7e797bf29a15005eb8d127848563254d6c0754 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 21 Jul 2022 13:25:41 +0200 Subject: [PATCH 38/61] update --- src/lightning_app/cli/lightning_cli.py | 15 ++++++++++----- src/lightning_app/utilities/commands/base.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index b9979a2735bc6..16d4ef1b7928c 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -1,13 +1,15 @@ import logging import os +import sys +from argparse import ArgumentParser from pathlib import Path -from typing import List, Tuple, Union, Optional +from typing import List, Optional, Tuple, Union from uuid import uuid4 -import sys + import click import requests from requests.exceptions import ConnectionError -from argparse import ArgumentParser + from lightning_app import __version__ as ver from lightning_app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init from lightning_app.core.constants import get_lightning_cloud_url, LOCAL_LAUNCH_ADMIN_VIEW @@ -40,6 +42,7 @@ def main(): else: app_command() + @click.group() @click.version_option(ver) def _main(): @@ -146,7 +149,9 @@ def app_command(): logger.warn("Lightning Commands are a beta feature and APIs aren't stable yet.") parser = ArgumentParser() - parser.add_argument("--app_id", default=None, type=Optional[str], help="Optional argument to identify an application.") + parser.add_argument( + "--app_id", default=None, type=Optional[str], help="Optional argument to identify an application." + ) hparams, _ = parser.parse_known_args() # 1: Collect the url and comments from the running application @@ -337,4 +342,4 @@ def _prepare_file(file: str) -> str: if exists: return file - raise FileNotFoundError(f"The provided file {file} hasn't been found.") \ No newline at end of file + raise FileNotFoundError(f"The provided file {file} hasn't been found.") diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 9d68b58bfda59..6e7c41ade01c4 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -102,7 +102,7 @@ def _download_command( # immutable commands for security concerns config = _ClientCommandConfig(**command_metadata) # tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") - #makedirs(tmpdir) + # makedirs(tmpdir) target_file = osp.join(".", f"{config.command}.py") if app_id: client = LightningClient() @@ -125,7 +125,7 @@ def _download_command( spec.loader.exec_module(mod) command = getattr(mod, cls_name)(method=None, requirements=config.requirements) models = {k: getattr(mod, v) for k, v in config.params.items()} - #shutil.rmtree(tmpdir) + # shutil.rmtree(tmpdir) return command, models From 27d0ca868aafbca1d2146ac25983c42dc9d3426b Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 21 Jul 2022 19:10:22 +0200 Subject: [PATCH 39/61] update --- src/lightning_app/cli/lightning_cli.py | 11 +++--- src/lightning_app/core/api.py | 37 ++++++++++++++------ src/lightning_app/utilities/commands/base.py | 20 ++++++----- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 16d4ef1b7928c..af27cc95acf6c 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -3,7 +3,7 @@ import sys from argparse import ArgumentParser from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Tuple, Union from uuid import uuid4 import click @@ -148,10 +148,10 @@ def app_command(): logger.warn("Lightning Commands are a beta feature and APIs aren't stable yet.") + debug_mode = bool(int(os.getenv("DEBUG", "0"))) + parser = ArgumentParser() - parser.add_argument( - "--app_id", default=None, type=Optional[str], help="Optional argument to identify an application." - ) + parser.add_argument("--app_id", default=None, type=str, help="Optional argument to identify an application.") hparams, _ = parser.parse_known_args() # 1: Collect the url and comments from the running application @@ -190,8 +190,9 @@ def app_command(): resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({})) assert resp.status_code == 200, resp.json() else: - client_command, models = _download_command(command_metadata, hparams.app_id) + client_command, models = _download_command(command_metadata, hparams.app_id, debug_mode=debug_mode) client_command._setup(metadata=command_metadata, models=models, app_url=url) + sys.argv = sys.argv[2:] client_command.run() diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index 11500baf443fe..f6e62bb0c2ddb 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -3,6 +3,8 @@ import os import queue import sys +import time +import traceback from copy import deepcopy from multiprocessing import Queue from threading import Event, Lock, Thread @@ -65,17 +67,21 @@ class SessionMiddleware: class UIRefresher(Thread): - def __init__(self, api_publish_state_queue, api_commands_metadata_queue) -> None: + def __init__(self, api_publish_state_queue, api_commands_metadata_queue, api_commands_responses_queue) -> None: super().__init__(daemon=True) self.api_publish_state_queue = api_publish_state_queue self.api_commands_metadata_queue = api_commands_metadata_queue + self.api_commands_responses_queue = api_commands_responses_queue self._exit_event = Event() def run(self): # TODO: Create multiple threads to handle the background logic # TODO: Investigate the use of `parallel=True` - while not self._exit_event.is_set(): - self.run_once() + try: + while not self._exit_event.is_set(): + self.run_once() + except Exception: + logger.error(traceback.print_exc()) def run_once(self): try: @@ -93,6 +99,14 @@ def run_once(self): except queue.Empty: pass + try: + response = self.api_commands_responses_queue.get(timeout=0) + with lock: + global commands_response_store + commands_response_store[response["id"]] = response + except queue.Empty: + pass + def join(self, timeout: Optional[float] = None) -> None: self._exit_event.set() super().join(timeout) @@ -176,18 +190,19 @@ async def run_remote_command( if not affiliation: raise Exception("The provided affiliation is empty.") - async def fn(request: Request): - data = await request.json() + async def fn(data): request_id = data["id"] api_commands_requests_queue.put(data) - resp = api_commands_responses_queue.get() - if request_id == resp["id"]: - return resp["response"] + t0 = time.time() + while request_id not in commands_response_store: + await asyncio.sleep(0.1) + if (time.time() - t0) > 15: + raise Exception("The response wasn't never received.") - raise Exception("This is a bug. It shouldn't happen.") + return commands_response_store[request_id] - return await asyncio.create_task(fn(request)) + return await asyncio.create_task(fn(data)) @fastapi_service.get("/api/v1/commands", response_class=JSONResponse) @@ -357,7 +372,7 @@ def start_server( global_app_state_store.add(TEST_SESSION_UUID) - refresher = UIRefresher(api_publish_state_queue, api_commands_metadata_queue) + refresher = UIRefresher(api_publish_state_queue, api_commands_metadata_queue, commands_responses_queue) refresher.setDaemon(True) refresher.start() diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 6e7c41ade01c4..65f0660c25cfa 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -96,14 +96,16 @@ def __call__(self, **kwargs: Any) -> Any: def _download_command( - command_metadata: Dict[str, Any], app_id: Optional[str] + command_metadata: Dict[str, Any], + app_id: Optional[str], + debug_mode: bool = False, ) -> Tuple[ClientCommand, Dict[str, BaseModel]]: # TODO: This is a skateboard implementation and the final version will rely on versioned # immutable commands for security concerns config = _ClientCommandConfig(**command_metadata) - # tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") - # makedirs(tmpdir) - target_file = osp.join(".", f"{config.command}.py") + tmpdir = osp.join(gettempdir(), f"{getuser()}_commands") + makedirs(tmpdir) + target_file = osp.join(tmpdir, f"{config.command}.py") if app_id: client = LightningClient() project_id = _get_project(client).project_id @@ -114,18 +116,18 @@ def _download_command( with open(target_file, "wb") as f: f.write(r.content) else: - shutil.copy(config.cls_path, target_file) - - breakpoint() + if not debug_mode: + shutil.copy(config.cls_path, target_file) cls_name = config.cls_name - spec = spec_from_file_location(config.cls_name, target_file) + spec = spec_from_file_location(config.cls_name, config.cls_path if debug_mode else target_file) mod = module_from_spec(spec) sys.modules[cls_name] = mod spec.loader.exec_module(mod) command = getattr(mod, cls_name)(method=None, requirements=config.requirements) models = {k: getattr(mod, v) for k, v in config.params.items()} - # shutil.rmtree(tmpdir) + if debug_mode: + shutil.rmtree(tmpdir) return command, models From bd6e449b70aab87e454e23e590f2d31c142a2bca Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 11:19:58 +0200 Subject: [PATCH 40/61] update --- src/lightning_app/core/constants.py | 2 +- src/lightning_app/runners/cloud.py | 4 ++-- .../utilities/packaging/lightning_utils.py | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/lightning_app/core/constants.py b/src/lightning_app/core/constants.py index 7644f60a2c50e..fd62de13cc013 100644 --- a/src/lightning_app/core/constants.py +++ b/src/lightning_app/core/constants.py @@ -22,7 +22,7 @@ REDIS_WARNING_QUEUE_SIZE = 1000 USER_ID = os.getenv("USER_ID", "1234") FRONTEND_DIR = os.path.join(os.path.dirname(lightning_app.__file__), "ui") -PREPARE_LIGHTING = bool(int(os.getenv("PREPARE_LIGHTING", "0"))) +PACKAGE_LIGHTNING = os.getenv("PACKAGE_LIGHTNING", None) LOCAL_LAUNCH_ADMIN_VIEW = bool(int(os.getenv("LOCAL_LAUNCH_ADMIN_VIEW", "0"))) CLOUD_UPLOAD_WARNING = int(os.getenv("CLOUD_UPLOAD_WARNING", "2")) DISABLE_DEPENDENCY_CACHE = bool(int(os.getenv("DISABLE_DEPENDENCY_CACHE", "0"))) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 37feda0b514f2..7213b8a66dd43 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -38,7 +38,7 @@ from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.dependency_caching import get_hash from lightning_app.utilities.packaging.app_config import AppConfig, find_config_file -from lightning_app.utilities.packaging.lightning_utils import _prepare_lightning_wheels_and_requirements +from lightning_app.utilities.packaging.lightning_utils import _PACKAGE_LIGHTNING_wheels_and_requirements logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def dispatch( config_file = find_config_file(self.entrypoint_file) app_config = AppConfig.load_from_file(config_file) if config_file else AppConfig() root = config_file.parent if config_file else Path(self.entrypoint_file).absolute().parent - cleanup_handle = _prepare_lightning_wheels_and_requirements(root) + cleanup_handle = _PACKAGE_LIGHTNING_wheels_and_requirements(root) repo = LocalSourceCodeDir(path=root) self._check_uploaded_folder(root, repo) requirements_file = root / "requirements.txt" diff --git a/src/lightning_app/utilities/packaging/lightning_utils.py b/src/lightning_app/utilities/packaging/lightning_utils.py index c851d753dd483..e980fd26ffeff 100644 --- a/src/lightning_app/utilities/packaging/lightning_utils.py +++ b/src/lightning_app/utilities/packaging/lightning_utils.py @@ -15,7 +15,7 @@ from lightning_app import _logger, _PROJECT_ROOT, _root_logger from lightning_app.__version__ import version -from lightning_app.core.constants import PREPARE_LIGHTING +from lightning_app.core.constants import PACKAGE_LIGHTNING from lightning_app.utilities.git import check_github_repository, get_dir_name logger = logging.getLogger(__name__) @@ -88,7 +88,7 @@ def get_dist_path_if_editable_install(project_name) -> str: return "" -def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable]: +def _PACKAGE_LIGHTNING_wheels_and_requirements(root: Path) -> Optional[Callable]: if "site-packages" in _PROJECT_ROOT: return @@ -96,11 +96,13 @@ def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable] # Packaging the Lightning codebase happens only inside the `lightning` repo. git_dir_name = get_dir_name() if check_github_repository() else None - if not PREPARE_LIGHTING and (not git_dir_name or (git_dir_name and not git_dir_name.startswith("lightning"))): + is_lightning = git_dir_name and git_dir_name == "lightning" + + if (PACKAGE_LIGHTNING is None and not is_lightning) or PACKAGE_LIGHTNING == "0": return - if not bool(int(os.getenv("SKIP_LIGHTING_WHEELS_BUILD", "0"))): - download_frontend(_PROJECT_ROOT) - _prepare_wheel(_PROJECT_ROOT) + + download_frontend(_PROJECT_ROOT) + _prepare_wheel(_PROJECT_ROOT) logger.info("Packaged Lightning with your application.") @@ -108,8 +110,8 @@ def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable] tar_files = [os.path.join(root, tar_name)] - # skipping this by default - if not bool(int(os.getenv("SKIP_LIGHTING_UTILITY_WHEELS_BUILD", "1"))): + # Don't skip by default + if (PACKAGE_LIGHTNING or is_lightning) and not bool(int(os.getenv("SKIP_LIGHTING_UTILITY_WHEELS_BUILD", "0"))): # building and copying launcher wheel if installed in editable mode launcher_project_path = get_dist_path_if_editable_install("lightning_launcher") if launcher_project_path: From 73a6a3fb35b2280b25a02e6386551c71dbd06e4b Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 14:02:08 +0200 Subject: [PATCH 41/61] update --- src/lightning_app/cli/lightning_cli.py | 6 +++--- src/lightning_app/source_code/uploader.py | 2 +- src/lightning_app/utilities/cli_helpers.py | 2 +- src/lightning_app/utilities/commands/base.py | 5 ++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index af27cc95acf6c..a816fa4af9373 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -152,7 +152,7 @@ def app_command(): parser = ArgumentParser() parser.add_argument("--app_id", default=None, type=str, help="Optional argument to identify an application.") - hparams, _ = parser.parse_known_args() + hparams, argv = parser.parse_known_args() # 1: Collect the url and comments from the running application url, commands = _retrieve_application_url_and_available_commands(hparams.app_id) @@ -162,7 +162,7 @@ def app_command(): if not commands: raise Exception("This application doesn't expose any commands yet.") - command = sys.argv[1] + command = argv[0] command_names = [c["command"] for c in commands] if command not in command_names: @@ -192,7 +192,7 @@ def app_command(): else: client_command, models = _download_command(command_metadata, hparams.app_id, debug_mode=debug_mode) client_command._setup(metadata=command_metadata, models=models, app_url=url) - sys.argv = sys.argv[2:] + sys.argv = argv[1:] client_command.run() diff --git a/src/lightning_app/source_code/uploader.py b/src/lightning_app/source_code/uploader.py index 7f3413d1e29b0..1262c74b8adfe 100644 --- a/src/lightning_app/source_code/uploader.py +++ b/src/lightning_app/source_code/uploader.py @@ -65,7 +65,7 @@ def upload_data(url: str, data, retries: int, disconnect_retry_wait_seconds: int retries = Retry(total=10) with requests.Session() as s: s.mount("https://", HTTPAdapter(max_retries=retries)) - if url.startswith("s3://"): + if "tar.gz?" in url: resp = s.put(url, data=data) if "ETag" not in resp.headers: raise ValueError(f"Unexpected response from S3, response: {resp.content}") diff --git a/src/lightning_app/utilities/cli_helpers.py b/src/lightning_app/utilities/cli_helpers.py index f3300deafa71d..fcce96ec64407 100644 --- a/src/lightning_app/utilities/cli_helpers.py +++ b/src/lightning_app/utilities/cli_helpers.py @@ -82,7 +82,7 @@ def _retrieve_application_url_and_available_commands(app_id_or_name_or_url: Opti lightningapp_names = [lightningapp.name for lightningapp in list_lightningapps.lightningapps] if not app_id_or_name_or_url: - raise Exception(f"Provide an application name, id or url with --id=X. Found {lightningapp_names}") + raise Exception(f"Provide an application name, id or url with --app_id=X. Found {lightningapp_names}") for lightningapp in list_lightningapps.lightningapps: if lightningapp.id == app_id_or_name_or_url or lightningapp.name == app_id_or_name_or_url: diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 65f0660c25cfa..4c3b427e59b58 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -61,7 +61,10 @@ def _setup(self, metadata: Dict[str, Any], models: Dict[str, BaseModel], app_url @property def state(self): if self._state is None: - self._state = AppState() + assert self.app_url + # TODO: Resolve this hack + os.environ["LIGHTNING_APP_STATE_URL"] = "1" + self._state = AppState(host=self.app_url) self._state._request_state() return self._state From 2e785bbf8afa8fd41c26f6c50f14709be30c7f1a Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 15:34:41 +0200 Subject: [PATCH 42/61] update --- src/lightning_app/cli/lightning_cli.py | 6 ++-- src/lightning_app/core/api.py | 2 +- src/lightning_app/utilities/commands/base.py | 2 +- tests/tests_app/utilities/test_commands.py | 37 +++++++++----------- 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index a816fa4af9373..80f4d161f499c 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -177,10 +177,10 @@ def app_command(): # 4: Execute commands if not command_metadata["is_client_command"]: - kwargs = {k.split("=")[0]: k.split("=")[1] for k in sys.argv[2:]} + kwargs = {k.split("=")[0].replace("--", ""): k.split("=")[1] for k in argv[1:]} for param in params: if param not in kwargs: - raise Exception(f"The argument --args {param}=X hasn't been provided.") + raise Exception(f"The argument --{param}=X hasn't been provided.") json = { "command_name": command, "command_arguments": kwargs, @@ -192,7 +192,7 @@ def app_command(): else: client_command, models = _download_command(command_metadata, hparams.app_id, debug_mode=debug_mode) client_command._setup(metadata=command_metadata, models=models, app_url=url) - sys.argv = argv[1:] + sys.argv = argv client_command.run() diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index f6e62bb0c2ddb..6cd6d8c65af20 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -103,7 +103,7 @@ def run_once(self): response = self.api_commands_responses_queue.get(timeout=0) with lock: global commands_response_store - commands_response_store[response["id"]] = response + commands_response_store[response["id"]] = response["response"] except queue.Empty: pass diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 4c3b427e59b58..66f1673df710f 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -180,7 +180,7 @@ def _upload_command(command_name: str, command: ClientCommand) -> Optional[str]: remote_url = str(shared_storage_path() / "artifacts" / filepath) fs = filesystem() - if _is_s3fs_available() and not fs.exists(remote_url): + if _is_s3fs_available(): from s3fs import S3FileSystem if not isinstance(fs, S3FileSystem): diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 5e217d2b30953..48527c8578983 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -1,15 +1,16 @@ +import argparse +import sys from multiprocessing import Process from time import sleep from unittest.mock import MagicMock import pytest import requests -from click.testing import CliRunner from pydantic import BaseModel from lightning import LightningFlow from lightning_app import LightningApp -from lightning_app.cli.lightning_cli import command +from lightning_app.cli.lightning_cli import app_command from lightning_app.core.constants import APP_SERVER_PORT from lightning_app.runners import MultiProcessRuntime from lightning_app.utilities.commands.base import _command_to_method_and_metadata, _download_command, ClientCommand @@ -18,12 +19,18 @@ class SweepConfig(BaseModel): sweep_name: str - num_trials: str + num_trials: int class SweepCommand(ClientCommand): - def run(self, sweep_name: str, num_trials: str) -> None: - config = SweepConfig(sweep_name=sweep_name, num_trials=num_trials) + def run(self) -> None: + print(sys.argv) + parser = argparse.ArgumentParser() + parser.add_argument("--sweep_name", type=str) + parser.add_argument("--num_trials", type=int) + hparams = parser.parse_args() + + config = SweepConfig(sweep_name=hparams.sweep_name, num_trials=hparams.num_trials) response = self.invoke_handler(config=config) assert response is True @@ -119,7 +126,7 @@ def target(): MultiProcessRuntime(app).dispatch() -def test_configure_commands(): +def test_configure_commands(monkeypatch): process = Process(target=target) process.start() time_left = 15 @@ -132,21 +139,11 @@ def test_configure_commands(): time_left -= 0.1 sleep(0.5) - runner = CliRunner() - result = runner.invoke( - command, - ["user_command", "--args", "name=something"], - catch_exceptions=False, - ) - assert result.exit_code == 0 + monkeypatch.setattr(sys, "argv", ["lightning", "user_command", "--name=something"]) + app_command() sleep(0.5) state = AppState() state._request_state() assert state.names == ["something"] - runner = CliRunner() - result = runner.invoke( - command, - ["sweep", "--args", "sweep_name=my_name", "--args", "num_trials=num_trials"], - catch_exceptions=False, - ) - assert result.exit_code == 0 + monkeypatch.setattr(sys, "argv", ["lightning", "sweep", "--sweep_name", "my_name", "--num_trials", "1"]) + app_command() From deb1c40e1b579cb51a34f98eef74aa60cc889fde Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 15:37:01 +0200 Subject: [PATCH 43/61] update --- tests/tests_app/core/test_lightning_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_app/core/test_lightning_api.py b/tests/tests_app/core/test_lightning_api.py index 4d228348e7340..9de7c63051b63 100644 --- a/tests/tests_app/core/test_lightning_api.py +++ b/tests/tests_app/core/test_lightning_api.py @@ -162,10 +162,11 @@ def test_update_publish_state_and_maybe_refresh_ui(): app = AppStageTestingApp(FlowA(), debug=True) publish_state_queue = MockQueue("publish_state_queue") commands_metadata_queue = MockQueue("commands_metadata_queue") + commands_responses_queue = MockQueue("commands_metadata_queue") publish_state_queue.put(app.state_with_changes) - thread = UIRefresher(publish_state_queue, commands_metadata_queue) + thread = UIRefresher(publish_state_queue, commands_metadata_queue, commands_responses_queue) thread.run_once() assert global_app_state_store.get_app_state("1234") == app.state_with_changes From 78dbd10839c89a56488a33d03d8bfe537bfee404 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 15:56:08 +0200 Subject: [PATCH 44/61] update --- MANIFEST.in | 6 ++++++ examples/app_commands/command.py | 9 +++++++-- setup.py | 2 +- tests/tests_app_examples/test_commands.py | 6 ++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..0600a541bec39 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,9 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src/lightning_app/cli/*-template * +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src/lightning_app/cli/*-template * diff --git a/examples/app_commands/command.py b/examples/app_commands/command.py index d0faa561195a9..74243ce59274e 100644 --- a/examples/app_commands/command.py +++ b/examples/app_commands/command.py @@ -1,3 +1,5 @@ +from argparse import ArgumentParser + from pydantic import BaseModel from lightning_app.utilities.commands import ClientCommand @@ -8,5 +10,8 @@ class CustomConfig(BaseModel): class CustomCommand(ClientCommand): - def run(self, name: str): - self.invoke_handler(config=CustomConfig(name=name)) + def run(self): + parser = ArgumentParser() + parser.add_argument("--name", type=str) + args = parser.parse_args() + self.invoke_handler(config=CustomConfig(name=args.name)) diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/tests/tests_app_examples/test_commands.py b/tests/tests_app_examples/test_commands.py index a30cdf1a4bb3f..5116b1b9d54bb 100644 --- a/tests/tests_app_examples/test_commands.py +++ b/tests/tests_app_examples/test_commands.py @@ -18,12 +18,14 @@ def test_commands_example_cloud() -> None: fetch_logs, ): app_id = admin_page.url.split("/")[-1] - cmd = f"lightning command trigger_with_client_command --args name=something --id {app_id}" + cmd = f"lightning trigger_with_client_command --name=something --app_id {app_id}" + Popen(cmd, shell=True).wait() + cmd = f"lightning trigger_without_client_command --name=else --app_id {app_id}" Popen(cmd, shell=True).wait() has_logs = False while not has_logs: for log in fetch_logs(): - if "['something']" in log: + if "['something', 'else']" in log: has_logs = True sleep(1) From da353a73b1db5e33e795864494fad85cfa6881fc Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:00:47 +0200 Subject: [PATCH 45/61] update --- src/lightning_app/utilities/packaging/lightning_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/utilities/packaging/lightning_utils.py b/src/lightning_app/utilities/packaging/lightning_utils.py index e980fd26ffeff..37f4ff22988eb 100644 --- a/src/lightning_app/utilities/packaging/lightning_utils.py +++ b/src/lightning_app/utilities/packaging/lightning_utils.py @@ -88,7 +88,7 @@ def get_dist_path_if_editable_install(project_name) -> str: return "" -def _PACKAGE_LIGHTNING_wheels_and_requirements(root: Path) -> Optional[Callable]: +def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable]: if "site-packages" in _PROJECT_ROOT: return From 05fc114c3139e3733564bbf35376a876bdd8d8f0 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:11:12 +0200 Subject: [PATCH 46/61] update --- MANIFEST.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0600a541bec39..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,9 +3,3 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src/lightning_app/cli/*-template * -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src/lightning_app/cli/*-template * From 4f001185227eb3407e542b30f2e8e033c1b9c91b Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:11:31 +0200 Subject: [PATCH 47/61] update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ From 4fee4ca6a7b777b88e811a8c425af144327e48f7 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:12:37 +0200 Subject: [PATCH 48/61] update --- src/lightning_app/core/flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/core/flow.py b/src/lightning_app/core/flow.py index d24033841d590..d1af891476a02 100644 --- a/src/lightning_app/core/flow.py +++ b/src/lightning_app/core/flow.py @@ -630,6 +630,6 @@ def my_remote_method(self, name): .. code-block:: bash - lightning command my_command_name --args name=my_own_name + lightning my_command_name --args name=my_own_name """ raise NotImplementedError From 886700518b72bb7e04aa5f56c813c199fc962b54 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:14:09 +0200 Subject: [PATCH 49/61] update --- src/lightning_app/source_code/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/source_code/uploader.py b/src/lightning_app/source_code/uploader.py index 1262c74b8adfe..f2c130e88b949 100644 --- a/src/lightning_app/source_code/uploader.py +++ b/src/lightning_app/source_code/uploader.py @@ -86,7 +86,7 @@ def upload(self) -> None: try: with open(self.source_file, "rb") as f: data = f.read() - self.upload_data(self.presigned_url, data, self.retries, self.disconnect_retry_wait_seconds) + self.upload_data(self.presigned_url, data, self.retries, self.disconnect_retry_wait_seconds) self.progress.update(task_id, advance=len(data)) finally: self.progress.stop() From 6e11661138ed200ed5f8a7ddbf6322b294cb31ec Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:22:56 +0200 Subject: [PATCH 50/61] update --- src/lightning_app/runners/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 7213b8a66dd43..37feda0b514f2 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -38,7 +38,7 @@ from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.dependency_caching import get_hash from lightning_app.utilities.packaging.app_config import AppConfig, find_config_file -from lightning_app.utilities.packaging.lightning_utils import _PACKAGE_LIGHTNING_wheels_and_requirements +from lightning_app.utilities.packaging.lightning_utils import _prepare_lightning_wheels_and_requirements logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def dispatch( config_file = find_config_file(self.entrypoint_file) app_config = AppConfig.load_from_file(config_file) if config_file else AppConfig() root = config_file.parent if config_file else Path(self.entrypoint_file).absolute().parent - cleanup_handle = _PACKAGE_LIGHTNING_wheels_and_requirements(root) + cleanup_handle = _prepare_lightning_wheels_and_requirements(root) repo = LocalSourceCodeDir(path=root) self._check_uploaded_folder(root, repo) requirements_file = root / "requirements.txt" From f62864197f817319b24bd851e5ff2a2a8f8bde0e Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 16:36:57 +0200 Subject: [PATCH 51/61] Update src/lightning_app/CHANGELOG.md Co-authored-by: Mansy --- src/lightning_app/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index 847e70b04f55c..7d0dcb589b9e3 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added -- Add support for `Lightning App Commands` through the `configure_commands` hook on the Lightning Flow and the `ClientCommand` ([#13602](https://github.com/PyTorchLightning/pytorch-lightning/pull/13602)) +- Add support for `Lightning App Commands` through the `configure_commands` hook on the Lightning Flow and the `ClientCommand` ([#13602](https://github.com/Lightning-AI/lightning/pull/13602)) ### Changed From bf237a4ef262d05f2f689f3abb9eb2541dd32312 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 17:06:42 +0200 Subject: [PATCH 52/61] update --- src/lightning_app/cli/lightning_cli.py | 2 +- src/lightning_app/source_code/uploader.py | 18 ++++++++---------- tests/tests_app/cli/test_cli.py | 8 ++++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 80f4d161f499c..c0a9cc5612155 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -37,7 +37,7 @@ def get_app_url(runtime_type: RuntimeType, *args) -> str: def main(): if len(sys.argv) == 1: _main() - elif sys.argv[1] in _main.commands.keys(): + elif sys.argv[1] in _main.commands.keys() or sys.argv[1] == "--help": _main() else: app_command() diff --git a/src/lightning_app/source_code/uploader.py b/src/lightning_app/source_code/uploader.py index f2c130e88b949..efbd050810e32 100644 --- a/src/lightning_app/source_code/uploader.py +++ b/src/lightning_app/source_code/uploader.py @@ -39,8 +39,7 @@ def __init__(self, presigned_url: str, source_file: str, total_size: int, name: self.total_size = total_size self.name = name - @staticmethod - def upload_data(url: str, data, retries: int, disconnect_retry_wait_seconds: int) -> str: + def upload_data(self, url: str, data: bytes, retries: int, disconnect_retry_wait_seconds: int) -> str: """Send data to url. Parameters @@ -65,20 +64,19 @@ def upload_data(url: str, data, retries: int, disconnect_retry_wait_seconds: int retries = Retry(total=10) with requests.Session() as s: s.mount("https://", HTTPAdapter(max_retries=retries)) - if "tar.gz?" in url: - resp = s.put(url, data=data) - if "ETag" not in resp.headers: - raise ValueError(f"Unexpected response from S3, response: {resp.content}") - return resp.headers["ETag"] - else: - resp = s.put(url, files={"file": data}) - return str(resp.status_code) + return self._upload_data(s, url, data) except BrokenPipeError: time.sleep(disconnect_retry_wait_seconds) disconnect_retries -= 1 raise ValueError("Unable to upload file after multiple attempts") + def _upload_data(self, s: requests.Session, url: str, data: bytes): + resp = s.put(url, data=data) + if "ETag" not in resp.headers: + raise ValueError(f"Unexpected response from S3, response: {resp.content}") + return resp.headers["ETag"] + def upload(self) -> None: """Upload files from source dir into target path in S3.""" task_id = self.progress.add_task("upload", filename=self.name, total=self.total_size) diff --git a/tests/tests_app/cli/test_cli.py b/tests/tests_app/cli/test_cli.py index 2626116990340..39d8d6b7890b6 100644 --- a/tests/tests_app/cli/test_cli.py +++ b/tests/tests_app/cli/test_cli.py @@ -5,7 +5,7 @@ from click.testing import CliRunner from lightning_cloud.openapi import Externalv1LightningappInstance -from lightning_app.cli.lightning_cli import get_app_url, login, logout, main, run +from lightning_app.cli.lightning_cli import _main, get_app_url, login, logout, run from lightning_app.runners.runtime_type import RuntimeType @@ -37,7 +37,7 @@ def test_start_target_url(runtime_type, extra_args, lightning_cloud_url, expecte assert get_app_url(runtime_type, *extra_args) == expected_url -@pytest.mark.parametrize("command", [main, run]) +@pytest.mark.parametrize("command", [_main, run]) def test_commands(command): runner = CliRunner() result = runner.invoke(command) @@ -46,12 +46,12 @@ def test_commands(command): def test_main_lightning_cli_help(): """Validate the Lightning CLI.""" - res = os.popen("python -m lightning_app --help").read() + res = os.popen("python -m lightning --help").read() assert "login " in res assert "logout " in res assert "run " in res - res = os.popen("python -m lightning_app run --help").read() + res = os.popen("python -m lightning run --help").read() assert "app " in res # hidden run commands should not appear in the help text From 6d011e77c662a32b498f9559328287a6df68aa00 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 17:33:29 +0200 Subject: [PATCH 53/61] update --- src/lightning_app/utilities/commands/base.py | 1 + tests/tests_app/components/python/test_python.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lightning_app/utilities/commands/base.py b/src/lightning_app/utilities/commands/base.py index 66f1673df710f..11661e51ca26a 100644 --- a/src/lightning_app/utilities/commands/base.py +++ b/src/lightning_app/utilities/commands/base.py @@ -66,6 +66,7 @@ def state(self): os.environ["LIGHTNING_APP_STATE_URL"] = "1" self._state = AppState(host=self.app_url) self._state._request_state() + os.environ.pop("LIGHTNING_APP_STATE_URL") return self._state def run(self, **cli_kwargs) -> None: diff --git a/tests/tests_app/components/python/test_python.py b/tests/tests_app/components/python/test_python.py index 283f449092d06..0ee59fda875a9 100644 --- a/tests/tests_app/components/python/test_python.py +++ b/tests/tests_app/components/python/test_python.py @@ -17,10 +17,9 @@ def test_non_existing_python_script(): run_work_isolated(python_script) assert not python_script.has_started - with pytest.raises(FileNotFoundError, match=match): - python_script = TracerPythonScript(match) - run_work_isolated(python_script) - assert not python_script.has_started + python_script = TracerPythonScript(match) + run_work_isolated(python_script) + assert python_script.has_failed def test_simple_python_script(): From 4485cf9b0fe920aa31de8766b9e1e5f7b787f068 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 17:50:25 +0200 Subject: [PATCH 54/61] update --- tests/tests_app/utilities/test_commands.py | 13 ++++++++++++- tests/tests_app/utilities/test_state.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 48527c8578983..636229ae0cdbd 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -39,12 +39,18 @@ class FlowCommands(LightningFlow): def __init__(self): super().__init__() self.names = [] + self.has_sweep = False + + def run(self): + if self.has_sweep and len(self.names) == 1: + sleep(2) + self._exit() def trigger_method(self, name: str): self.names.append(name) def sweep(self, config: SweepConfig): - print(config) + self.has_sweep = True return True def configure_commands(self): @@ -147,3 +153,8 @@ def test_configure_commands(monkeypatch): assert state.names == ["something"] monkeypatch.setattr(sys, "argv", ["lightning", "sweep", "--sweep_name", "my_name", "--num_trials", "1"]) app_command() + time_left = 15 + while process.exitcode is None: + sleep(0.1) + time_left -= 0.1 + assert process.exitcode == 0 diff --git a/tests/tests_app/utilities/test_state.py b/tests/tests_app/utilities/test_state.py index e275817f680fc..0740ffc615b87 100644 --- a/tests/tests_app/utilities/test_state.py +++ b/tests/tests_app/utilities/test_state.py @@ -15,7 +15,7 @@ def test_app_state_not_connected(_): """Test an error message when a disconnected AppState tries to access attributes.""" - state = AppState() + state = AppState(port=8000) with pytest.raises(AttributeError, match="Failed to connect and fetch the app state"): _ = state.value with pytest.raises(AttributeError, match="Failed to connect and fetch the app state"): @@ -209,7 +209,7 @@ def test_attach_plugin(): @mock.patch("lightning_app.utilities.state._configure_session", return_value=requests) def test_app_state_connection_error(_): """Test an error message when a connection to retrieve the state can't be established.""" - app_state = AppState() + app_state = AppState(port=8000) with pytest.raises(AttributeError, match=r"Failed to connect and fetch the app state\. Is the app running?"): app_state._request_state() From 5f5c8bf5284bcd457d3af135a349af140fd21a30 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 17:50:40 +0200 Subject: [PATCH 55/61] update --- tests/tests_app/utilities/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 636229ae0cdbd..30f4197aa3173 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -154,7 +154,7 @@ def test_configure_commands(monkeypatch): monkeypatch.setattr(sys, "argv", ["lightning", "sweep", "--sweep_name", "my_name", "--num_trials", "1"]) app_command() time_left = 15 - while process.exitcode is None: + while time_left > 0 or process.exitcode is None: sleep(0.1) time_left -= 0.1 assert process.exitcode == 0 From 7e6d09b150f5a4abac609b0c67f0a925100e5351 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 18:01:23 +0200 Subject: [PATCH 56/61] update --- .github/workflows/ci-app_cloud_e2e_test.yml | 2 +- examples/app_commands/command.py | 2 +- src/lightning_app/core/app.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-app_cloud_e2e_test.yml b/.github/workflows/ci-app_cloud_e2e_test.yml index 81823bb517804..cb0fbdf40a9e0 100644 --- a/.github/workflows/ci-app_cloud_e2e_test.yml +++ b/.github/workflows/ci-app_cloud_e2e_test.yml @@ -156,7 +156,7 @@ jobs: shell: bash run: | mkdir -p ${VIDEO_LOCATION} - HEADLESS=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes + HEADLESS=1 PACKAGE_LIGHTNING=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes # Delete the artifacts if successful rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }} diff --git a/examples/app_commands/command.py b/examples/app_commands/command.py index 74243ce59274e..8c3070f6d764c 100644 --- a/examples/app_commands/command.py +++ b/examples/app_commands/command.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from lightning_app.utilities.commands import ClientCommand +from lightning.app.utilities.commands import ClientCommand class CustomConfig(BaseModel): diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 6599b53efcb95..8705adb562e8b 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -11,12 +11,12 @@ from deepdiff import DeepDiff, Delta import lightning_app +from lightning.app.utilities.commands.base import _populate_commands_endpoint, _process_command_requests from lightning_app.core.constants import FLOW_DURATION_SAMPLES, FLOW_DURATION_THRESHOLD, STATE_ACCUMULATE_WAIT from lightning_app.core.queues import BaseQueue, SingleProcessQueue from lightning_app.frontend import Frontend from lightning_app.storage.path import storage_root_dir from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef -from lightning_app.utilities.commands.base import _populate_commands_endpoint, _process_command_requests from lightning_app.utilities.component import _convert_paths_after_init from lightning_app.utilities.enum import AppStage from lightning_app.utilities.exceptions import CacheMissException, ExitAppException From 30e84e81e7c6ca113a3601fa56330ef1f01c965e Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 18:07:55 +0200 Subject: [PATCH 57/61] update --- examples/app_commands/app.py | 4 ++-- src/lightning_app/cli/lightning_cli.py | 6 ++---- src/lightning_app/core/api.py | 5 +++-- src/lightning_app/core/app.py | 2 +- src/lightning_app/source_code/uploader.py | 2 +- tests/tests_app/source_code/test_uploader.py | 5 +++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/app_commands/app.py b/examples/app_commands/app.py index 20e027c779289..99eb15c75c709 100644 --- a/examples/app_commands/app.py +++ b/examples/app_commands/app.py @@ -29,11 +29,11 @@ def trigger_with_client_command(self, config: CustomConfig): self.names.append(config.name) def configure_commands(self): - commnands = [ + commands = [ {"trigger_without_client_command": self.trigger_without_client_command}, {"trigger_with_client_command": CustomCommand(self.trigger_with_client_command)}, ] - return commnands + self.child_flow.configure_commands() + return commands + self.child_flow.configure_commands() app = LightningApp(FlowCommands()) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index c0a9cc5612155..74b2d1c4926e1 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -172,11 +172,9 @@ def app_command(): command_metadata = [c for c in commands if c["command"] == command][0] params = command_metadata["params"] - # 3: Prepare the arguments provided by the users. - # TODO: Improve what is supported there. - - # 4: Execute commands + # 3: Execute the command if not command_metadata["is_client_command"]: + # TODO: Improve what is supported there. kwargs = {k.split("=")[0].replace("--", ""): k.split("=")[1] for k in argv[1:]} for param in params: if param not in kwargs: diff --git a/src/lightning_app/core/api.py b/src/lightning_app/core/api.py index 6cd6d8c65af20..f38c1844e28e0 100644 --- a/src/lightning_app/core/api.py +++ b/src/lightning_app/core/api.py @@ -80,8 +80,9 @@ def run(self): try: while not self._exit_event.is_set(): self.run_once() - except Exception: + except Exception as e: logger.error(traceback.print_exc()) + raise e def run_once(self): try: @@ -198,7 +199,7 @@ async def fn(data): while request_id not in commands_response_store: await asyncio.sleep(0.1) if (time.time() - t0) > 15: - raise Exception("The response wasn't never received.") + raise Exception("The response was never received.") return commands_response_store[request_id] diff --git a/src/lightning_app/core/app.py b/src/lightning_app/core/app.py index 8705adb562e8b..6599b53efcb95 100644 --- a/src/lightning_app/core/app.py +++ b/src/lightning_app/core/app.py @@ -11,12 +11,12 @@ from deepdiff import DeepDiff, Delta import lightning_app -from lightning.app.utilities.commands.base import _populate_commands_endpoint, _process_command_requests from lightning_app.core.constants import FLOW_DURATION_SAMPLES, FLOW_DURATION_THRESHOLD, STATE_ACCUMULATE_WAIT from lightning_app.core.queues import BaseQueue, SingleProcessQueue from lightning_app.frontend import Frontend from lightning_app.storage.path import storage_root_dir from lightning_app.utilities.app_helpers import _delta_to_appstate_delta, _LightningAppRef +from lightning_app.utilities.commands.base import _populate_commands_endpoint, _process_command_requests from lightning_app.utilities.component import _convert_paths_after_init from lightning_app.utilities.enum import AppStage from lightning_app.utilities.exceptions import CacheMissException, ExitAppException diff --git a/src/lightning_app/source_code/uploader.py b/src/lightning_app/source_code/uploader.py index efbd050810e32..5816c01c3fb1c 100644 --- a/src/lightning_app/source_code/uploader.py +++ b/src/lightning_app/source_code/uploader.py @@ -74,7 +74,7 @@ def upload_data(self, url: str, data: bytes, retries: int, disconnect_retry_wait def _upload_data(self, s: requests.Session, url: str, data: bytes): resp = s.put(url, data=data) if "ETag" not in resp.headers: - raise ValueError(f"Unexpected response from S3, response: {resp.content}") + raise ValueError(f"Unexpected response from {url}, response: {resp.content}") return resp.headers["ETag"] def upload(self) -> None: diff --git a/tests/tests_app/source_code/test_uploader.py b/tests/tests_app/source_code/test_uploader.py index 82789e83e37a9..774442291deed 100644 --- a/tests/tests_app/source_code/test_uploader.py +++ b/tests/tests_app/source_code/test_uploader.py @@ -39,10 +39,11 @@ def test_file_uploader(): @mock.patch("lightning_app.source_code.uploader.requests.Session", MockedRequestSession) def test_file_uploader_failing_when_no_etag(): response["response"] = MagicMock(headers={}) + presigned_url = "https://test-url" file_uploader = uploader.FileUploader( - presigned_url="https://test-url", source_file="test.txt", total_size=100, name="test.txt" + presigned_url=presigned_url, source_file="test.txt", total_size=100, name="test.txt" ) file_uploader.progress = MagicMock() - with pytest.raises(ValueError, match="Unexpected response from S3, response"): + with pytest.raises(ValueError, match=f"Unexpected response from {presigned_url}, response"): file_uploader.upload() From 1abef079b1494781458c188d31c7243cf690622a Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Jul 2022 18:27:58 +0200 Subject: [PATCH 58/61] update --- MANIFEST.in | 3 +++ setup.py | 2 +- tests/tests_app/components/python/test_python.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a8dbcff69b631..c22e9b09d4985 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,6 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info +recursive-include src *.md +recursive-include requirements *.txt +recursive-include src/lightning_app/cli/*-template * diff --git a/setup.py b/setup.py index a542b3c1e0291..6d271cc40b0aa 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") +_PACKAGE_NAME = "" _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/tests/tests_app/components/python/test_python.py b/tests/tests_app/components/python/test_python.py index 0ee59fda875a9..61969ef1c4c51 100644 --- a/tests/tests_app/components/python/test_python.py +++ b/tests/tests_app/components/python/test_python.py @@ -17,7 +17,7 @@ def test_non_existing_python_script(): run_work_isolated(python_script) assert not python_script.has_started - python_script = TracerPythonScript(match) + python_script = TracerPythonScript(match, raise_exception=False) run_work_isolated(python_script) assert python_script.has_failed From 495c9f0c9bc0258698c4ca1e81ec7e11e91eb349 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 25 Jul 2022 15:28:19 +0200 Subject: [PATCH 59/61] update --- MANIFEST.in | 3 --- 1 file changed, 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index c22e9b09d4985..a8dbcff69b631 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,3 @@ exclude requirements.txt exclude __pycache__ include .actions/setup_tools.py include *.cff # citation info -recursive-include src *.md -recursive-include requirements *.txt -recursive-include src/lightning_app/cli/*-template * From 1d61f142e542bca7035d609f05d38e060559781d Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 25 Jul 2022 18:26:58 +0200 Subject: [PATCH 60/61] update --- tests/tests_app/utilities/test_commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 30f4197aa3173..1e8e36ed09545 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -13,6 +13,7 @@ from lightning_app.cli.lightning_cli import app_command from lightning_app.core.constants import APP_SERVER_PORT from lightning_app.runners import MultiProcessRuntime +from lightning_app.testing.helpers import RunIf from lightning_app.utilities.commands.base import _command_to_method_and_metadata, _download_command, ClientCommand from lightning_app.utilities.state import AppState @@ -89,6 +90,7 @@ def run_failure_2(name: CustomModel): pass +@RunIf(skip_windows=True) def test_command_to_method_and_metadata(): with pytest.raises(Exception, match="The provided annotation for the argument name"): _command_to_method_and_metadata(ClientCommand(run_failure_0)) From 5b6434d3e6c40cb973c98ac128195016c9076d84 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 25 Jul 2022 18:37:58 +0200 Subject: [PATCH 61/61] update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d271cc40b0aa..a542b3c1e0291 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ from setuptools import setup -_PACKAGE_NAME = "" +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/