[App] Introduce Commands (#13602)

Author: tchaton

.github/workflows/ci-app_cloud_e2e_test.yml

@@ -54,6 +54,7 @@ jobs:
           - custom_work_dependencies
           - drive
           - payload
+          - commands
     timeout-minutes: 35
     steps:
       - uses: actions/checkout@v2
@@ -155,7 +156,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p ${VIDEO_LOCATION}
-          HEADLESS=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes
+          HEADLESS=1 PACKAGE_LIGHTNING=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes
           # Delete the artifacts if successful
           rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }}
 

.gitignore

@@ -109,6 +109,7 @@ celerybeat-schedule
 
 # dotenv
 .env
+.env_stagging
 
 # virtualenv
 .venv
@@ -160,3 +161,5 @@ tags
 .tags
 src/lightning_app/ui/*
 *examples/template_react_ui*
+hars*
+artifacts/*

examples/app_commands/.lightning

@@ -0,0 +1 @@
+name: app-commands

examples/app_commands/app.py

@@ -0,0 +1,39 @@
+from command import CustomCommand, CustomConfig
+
+from lightning import LightningFlow
+from lightning_app.core.app import LightningApp
+
+
+class ChildFlow(LightningFlow):
+    def trigger_method(self, name: str):
+        print(f"Hello {name}")
+
+    def configure_commands(self):
+        return [{"nested_trigger_command": self.trigger_method}]
+
+
+class FlowCommands(LightningFlow):
+    def __init__(self):
+        super().__init__()
+        self.names = []
+        self.child_flow = ChildFlow()
+
+    def run(self):
+        if len(self.names):
+            print(self.names)
+
+    def trigger_without_client_command(self, name: str):
+        self.names.append(name)
+
+    def trigger_with_client_command(self, config: CustomConfig):
+        self.names.append(config.name)
+
+    def configure_commands(self):
+        commands = [
+            {"trigger_without_client_command": self.trigger_without_client_command},
+            {"trigger_with_client_command": CustomCommand(self.trigger_with_client_command)},
+        ]
+        return commands + self.child_flow.configure_commands()
+
+
+app = LightningApp(FlowCommands())

examples/app_commands/command.py

@@ -0,0 +1,17 @@
+from argparse import ArgumentParser
+
+from pydantic import BaseModel
+
+from lightning.app.utilities.commands import ClientCommand
+
+
+class CustomConfig(BaseModel):
+    name: str
+
+
+class CustomCommand(ClientCommand):
+    def run(self):
+        parser = ArgumentParser()
+        parser.add_argument("--name", type=str)
+        args = parser.parse_args()
+        self.invoke_handler(config=CustomConfig(name=args.name))

src/lightning_app/CHANGELOG.md

@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Add support for `Lightning App Commands` through the `configure_commands` hook on the Lightning Flow and the `ClientCommand`  ([#13602](https://github.com/Lightning-AI/lightning/pull/13602))
+
+### Changed
+
 - Update the Lightning App docs ([#13537](https://github.com/PyTorchLightning/pytorch-lightning/pull/13537))
 
 ### Changed

src/lightning_app/cli/lightning_cli.py

@@ -1,19 +1,27 @@
 import logging
 import os
+import sys
+from argparse import ArgumentParser
 from pathlib import Path
 from typing import List, Tuple, Union
+from uuid import uuid4
 
 import click
+import requests
 from requests.exceptions import ConnectionError
 
 from lightning_app import __version__ as ver
 from lightning_app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init
 from lightning_app.core.constants import get_lightning_cloud_url, LOCAL_LAUNCH_ADMIN_VIEW
 from lightning_app.runners.runtime import dispatch
 from lightning_app.runners.runtime_type import RuntimeType
-from lightning_app.utilities.cli_helpers import _format_input_env_variables
+from lightning_app.utilities.cli_helpers import (
+    _format_input_env_variables,
+    _retrieve_application_url_and_available_commands,
+)
 from lightning_app.utilities.install_components import register_all_external_components
 from lightning_app.utilities.login import Auth
+from lightning_app.utilities.state import headers_for
 
 logger = logging.getLogger(__name__)
 
@@ -26,14 +34,23 @@ def get_app_url(runtime_type: RuntimeType, *args) -> str:
         return "http://127.0.0.1:7501/admin" if LOCAL_LAUNCH_ADMIN_VIEW else "http://127.0.0.1:7501/view"
 
 
+def main():
+    if len(sys.argv) == 1:
+        _main()
+    elif sys.argv[1] in _main.commands.keys() or sys.argv[1] == "--help":
+        _main()
+    else:
+        app_command()
+
+
 @click.group()
 @click.version_option(ver)
-def main():
+def _main():
     register_all_external_components()
     pass
 
 
-@main.command()
+@_main.command()
 def login():
     """Log in to your Lightning.ai account."""
     auth = Auth()
@@ -46,7 +63,7 @@ def login():
         exit(1)
 
 
-@main.command()
+@_main.command()
 def logout():
     """Log out of your Lightning.ai account."""
     Auth().clear()
@@ -93,7 +110,7 @@ def on_before_run(*args):
         click.echo("Application is ready in the cloud")
 
 
-@main.group()
+@_main.group()
 def run():
     """Run your application."""
 
@@ -125,31 +142,83 @@ def run_app(
     _run_app(file, cloud, without_server, no_cache, name, blocking, open_ui, env)
 
 
-@main.group(hidden=True)
+def app_command():
+    """Execute a function in a running application from its name."""
+    from lightning_app.utilities.commands.base import _download_command
+
+    logger.warn("Lightning Commands are a beta feature and APIs aren't stable yet.")
+
+    debug_mode = bool(int(os.getenv("DEBUG", "0")))
+
+    parser = ArgumentParser()
+    parser.add_argument("--app_id", default=None, type=str, help="Optional argument to identify an application.")
+    hparams, argv = parser.parse_known_args()
+
+    # 1: Collect the url and comments from the running application
+    url, commands = _retrieve_application_url_and_available_commands(hparams.app_id)
+    if url is None or commands is None:
+        raise Exception("We couldn't find any matching running app.")
+
+    if not commands:
+        raise Exception("This application doesn't expose any commands yet.")
+
+    command = argv[0]
+
+    command_names = [c["command"] for c in commands]
+    if command not in command_names:
+        raise Exception(f"The provided command {command} isn't available in {command_names}")
+
+    # 2: Send the command from the user
+    command_metadata = [c for c in commands if c["command"] == command][0]
+    params = command_metadata["params"]
+
+    # 3: Execute the command
+    if not command_metadata["is_client_command"]:
+        # TODO: Improve what is supported there.
+        kwargs = {k.split("=")[0].replace("--", ""): k.split("=")[1] for k in argv[1:]}
+        for param in params:
+            if param not in kwargs:
+                raise Exception(f"The argument --{param}=X hasn't been provided.")
+        json = {
+            "command_name": command,
+            "command_arguments": kwargs,
+            "affiliation": command_metadata["affiliation"],
+            "id": str(uuid4()),
+        }
+        resp = requests.post(url + "/api/v1/commands", json=json, headers=headers_for({}))
+        assert resp.status_code == 200, resp.json()
+    else:
+        client_command, models = _download_command(command_metadata, hparams.app_id, debug_mode=debug_mode)
+        client_command._setup(metadata=command_metadata, models=models, app_url=url)
+        sys.argv = argv
+        client_command.run()
+
+
+@_main.group(hidden=True)
 def fork():
     """Fork an application."""
     pass
 
 
-@main.group(hidden=True)
+@_main.group(hidden=True)
 def stop():
     """Stop your application."""
     pass
 
 
-@main.group(hidden=True)
+@_main.group(hidden=True)
 def delete():
     """Delete an application."""
     pass
 
 
-@main.group(name="list", hidden=True)
+@_main.group(name="list", hidden=True)
 def get_list():
     """List your applications."""
     pass
 
 
-@main.group()
+@_main.group()
 def install():
     """Install Lightning apps and components."""
 
@@ -207,7 +276,7 @@ def install_component(name, yes, version):
         cmd_install.gallery_component(name, yes, version)
 
 
-@main.group()
+@_main.group()
 def init():
     """Init a Lightning app and component."""
 

src/lightning_app/components/python/tracer.py

@@ -93,8 +93,6 @@ def __init__(
             :language: python
         """
         super().__init__(**kwargs)
-        if not os.path.exists(script_path):
-            raise FileNotFoundError(f"The provided `script_path` {script_path}` wasn't found.")
         self.script_path = str(script_path)
         if isinstance(script_args, str):
             script_args = script_args.split(" ")
@@ -105,6 +103,8 @@ def __init__(
             setattr(self, name, None)
 
     def run(self, **kwargs):
+        if not os.path.exists(self.script_path):
+            raise FileNotFoundError(f"The provided `script_path` {self.script_path}` wasn't found.")
         kwargs = {k: v.value if isinstance(v, Payload) else v for k, v in kwargs.items()}
         init_globals = globals()
         init_globals.update(kwargs)