Merge pull request #1546 from janhq/j/engine-management

feat: engine management

Author: namchuai

docs/static/openapi/cortex.json

@@ -73,7 +73,6 @@ find_package(jsoncpp CONFIG REQUIRED)
 find_package(Drogon CONFIG REQUIRED)
 find_package(yaml-cpp CONFIG REQUIRED)
 find_package(httplib CONFIG REQUIRED)
-find_package(nlohmann_json CONFIG REQUIRED)
 find_package(unofficial-minizip CONFIG REQUIRED)
 find_package(LibArchive REQUIRED)
 find_package(CURL REQUIRED)
@@ -149,7 +148,6 @@ add_executable(${TARGET_NAME} main.cc
 target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
 target_link_libraries(${TARGET_NAME} PRIVATE httplib::httplib)
-target_link_libraries(${TARGET_NAME} PRIVATE nlohmann_json::nlohmann_json)
 target_link_libraries(${TARGET_NAME} PRIVATE unofficial::minizip::minizip)
 target_link_libraries(${TARGET_NAME} PRIVATE LibArchive::LibArchive)
 target_link_libraries(${TARGET_NAME} PRIVATE CURL::libcurl)

engine/CMakeLists.txt

@@ -63,7 +63,6 @@ add_compile_definitions(CORTEX_CONFIG_FILE_PATH="${CORTEX_CONFIG_FILE_PATH}")
 find_package(jsoncpp CONFIG REQUIRED)
 find_package(yaml-cpp CONFIG REQUIRED)
 find_package(httplib CONFIG REQUIRED)
-find_package(nlohmann_json CONFIG REQUIRED)
 find_package(CLI11 CONFIG REQUIRED)
 find_package(unofficial-minizip CONFIG REQUIRED)
 find_package(LibArchive REQUIRED)
@@ -87,7 +86,6 @@ add_executable(${TARGET_NAME} main.cc
   )
 
 target_link_libraries(${TARGET_NAME} PRIVATE httplib::httplib)
-target_link_libraries(${TARGET_NAME} PRIVATE nlohmann_json::nlohmann_json)
 target_link_libraries(${TARGET_NAME} PRIVATE CLI11::CLI11)
 target_link_libraries(${TARGET_NAME} PRIVATE unofficial::minizip::minizip)
 target_link_libraries(${TARGET_NAME} PRIVATE LibArchive::LibArchive)
@@ -128,4 +126,4 @@ set_target_properties(${TARGET_NAME} PROPERTIES
                       RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}
                       RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}
                       RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
-)
+)

engine/cli/CMakeLists.txt

@@ -1,12 +1,14 @@
 #include "command_line_parser.h"
 #include <memory>
 #include <optional>
+#include <string>
 #include "commands/cortex_upd_cmd.h"
 #include "commands/engine_get_cmd.h"
 #include "commands/engine_install_cmd.h"
 #include "commands/engine_list_cmd.h"
 #include "commands/engine_uninstall_cmd.h"
-#include "commands/model_alias_cmd.h"
+#include "commands/engine_update_cmd.h"
+#include "commands/engine_use_cmd.h"
 #include "commands/model_del_cmd.h"
 #include "commands/model_get_cmd.h"
 #include "commands/model_import_cmd.h"
@@ -94,8 +96,8 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
           CLI_LOG("\nNew Cortex release available: "
                   << CORTEX_CPP_VERSION << " -> " << *latest_version);
           CLI_LOG("To update, run: " << commands::GetRole()
-                                      << commands::GetCortexBinary()
-                                      << " update");
+                                     << commands::GetCortexBinary()
+                                     << " update");
         }
         done = true;
       });
@@ -138,7 +140,8 @@ void CommandLineParser::SetupCommonCommands() {
     }
   });
 
-  auto run_cmd = app_.add_subcommand("run", "Shortcut: pull, start & chat with a model");
+  auto run_cmd =
+      app_.add_subcommand("run", "Shortcut: pull, start & chat with a model");
   run_cmd->group(kCommonCommandsGroup);
   run_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
                  " run [options] [model_id]");
@@ -270,30 +273,6 @@ void CommandLineParser::SetupModelCommands() {
                                  cml_data_.model_id);
   });
 
-  std::string model_alias;
-  auto model_alias_cmd =
-      models_cmd->add_subcommand("alias", "Add a model alias instead of ID");
-  model_alias_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
-                         " models alias --model_id [model_id] --alias [alias]");
-  model_alias_cmd->group(kSubcommands);
-  model_alias_cmd->add_option(
-      "--model_id", cml_data_.model_id,
-      "Can be a model ID or model alias");
-  model_alias_cmd->add_option("--alias", cml_data_.model_alias,
-                              "new alias to be set");
-  model_alias_cmd->callback([this, model_alias_cmd]() {
-    if (std::exchange(executed_, true))
-      return;
-    if (cml_data_.model_id.empty() || cml_data_.model_alias.empty()) {
-      CLI_LOG("[model_id] and [alias] are required\n");
-      CLI_LOG(model_alias_cmd->help());
-      return;
-    }
-    commands::ModelAliasCmd mdc;
-    mdc.Exec(cml_data_.config.apiServerHost,
-             std::stoi(cml_data_.config.apiServerPort), cml_data_.model_id,
-             cml_data_.model_alias);
-  });
   // Model update parameters comment
   ModelUpdate(models_cmd);
 
@@ -384,6 +363,41 @@ void CommandLineParser::SetupEngineCommands() {
     EngineUninstall(uninstall_cmd, engine_name);
   }
 
+  auto engine_upd_cmd = engines_cmd->add_subcommand("update", "Update engine");
+  engine_upd_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                        " engines update [engine_name]");
+  engine_upd_cmd->callback([this, engine_upd_cmd] {
+    if (std::exchange(executed_, true))
+      return;
+    if (engine_upd_cmd->get_subcommands().empty()) {
+      CLI_LOG("[engine_name] is required\n");
+      CLI_LOG(engine_upd_cmd->help());
+    }
+  });
+  engine_upd_cmd->group(kSubcommands);
+  for (auto& engine : engine_service_.kSupportEngines) {
+    std::string engine_name{engine};
+    EngineUpdate(engine_upd_cmd, engine_name);
+  }
+
+  auto engine_use_cmd =
+      engines_cmd->add_subcommand("use", "Set engine as default");
+  engine_use_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                        " engines use [engine_name]");
+  engine_use_cmd->callback([this, engine_use_cmd] {
+    if (std::exchange(executed_, true))
+      return;
+    if (engine_use_cmd->get_subcommands().empty()) {
+      CLI_LOG("[engine_name] is required\n");
+      CLI_LOG(engine_use_cmd->help());
+    }
+  });
+  engine_use_cmd->group(kSubcommands);
+  for (auto& engine : engine_service_.kSupportEngines) {
+    std::string engine_name{engine};
+    EngineUse(engine_use_cmd, engine_name);
+  }
+
   EngineGet(engines_cmd);
 }
 
@@ -400,7 +414,11 @@ void CommandLineParser::SetupSystemCommands() {
                                             << " to " << cml_data_.port);
       auto config_path = file_manager_utils::GetConfigurationPath();
       cml_data_.config.apiServerPort = std::to_string(cml_data_.port);
-      config_yaml_utils::DumpYamlConfig(cml_data_.config, config_path.string());
+      auto result = config_yaml_utils::DumpYamlConfig(cml_data_.config,
+                                                      config_path.string());
+      if (result.has_error()) {
+        CLI_LOG("Error update " << config_path.string() << result.error());
+      }
     }
     commands::ServerStartCmd ssc;
     ssc.Exec(cml_data_.config.apiServerHost,
@@ -417,8 +435,7 @@ void CommandLineParser::SetupSystemCommands() {
     ssc.Exec();
   });
 
-  auto ps_cmd =
-      app_.add_subcommand("ps", "Show active model statuses");
+  auto ps_cmd = app_.add_subcommand("ps", "Show active model statuses");
   ps_cmd->group(kSystemGroup);
   ps_cmd->usage("Usage:\n" + commands::GetCortexBinary() + "ps");
   ps_cmd->callback([&]() {
@@ -460,13 +477,16 @@ void CommandLineParser::EngineInstall(CLI::App* parent,
   install_engine_cmd->add_option("-s, --source", src,
                                  "Install engine by local path");
 
+  install_engine_cmd->add_flag("-m, --menu", cml_data_.show_menu,
+                               "Display menu for engine variant selection");
+
   install_engine_cmd->callback([this, engine_name, &version, &src] {
     if (std::exchange(executed_, true))
       return;
     try {
-      commands::EngineInstallCmd(download_service_,
-                                 cml_data_.config.apiServerHost,
-                                 std::stoi(cml_data_.config.apiServerPort))
+      commands::EngineInstallCmd(
+          download_service_, cml_data_.config.apiServerHost,
+          std::stoi(cml_data_.config.apiServerPort), cml_data_.show_menu)
           .Exec(engine_name, version, src);
     } catch (const std::exception& e) {
       CTL_ERR(e.what());
@@ -494,6 +514,47 @@ void CommandLineParser::EngineUninstall(CLI::App* parent,
   });
 }
 
+void CommandLineParser::EngineUpdate(CLI::App* parent,
+                                     const std::string& engine_name) {
+  auto engine_update_cmd = parent->add_subcommand(engine_name, "");
+  engine_update_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                           " engines update " + engine_name);
+  engine_update_cmd->group(kEngineGroup);
+
+  engine_update_cmd->callback([this, engine_name] {
+    if (std::exchange(executed_, true))
+      return;
+    try {
+      commands::EngineUpdateCmd().Exec(
+          cml_data_.config.apiServerHost,
+          std::stoi(cml_data_.config.apiServerPort), engine_name);
+    } catch (const std::exception& e) {
+      CTL_ERR(e.what());
+    }
+  });
+}
+
+void CommandLineParser::EngineUse(CLI::App* parent,
+                                  const std::string& engine_name) {
+  auto engine_use_cmd = parent->add_subcommand(engine_name, "");
+  engine_use_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                        " engines use " + engine_name);
+  engine_use_cmd->group(kEngineGroup);
+
+  engine_use_cmd->callback([this, engine_name] {
+    if (std::exchange(executed_, true))
+      return;
+    auto result = commands::EngineUseCmd().Exec(
+        cml_data_.config.apiServerHost,
+        std::stoi(cml_data_.config.apiServerPort), engine_name);
+    if (result.has_error()) {
+      CTL_ERR(result.error());
+    } else {
+      CTL_INF("Engine " << engine_name << " is set as default");
+    }
+  });
+}
+
 void CommandLineParser::EngineGet(CLI::App* parent) {
   auto get_cmd = parent->add_subcommand("get", "Get engine info");
   get_cmd->usage("Usage:\n" + commands::GetCortexBinary() +

engine/cli/command_line_parser.cc

@@ -27,7 +27,12 @@ class CommandLineParser {
 
   void EngineUninstall(CLI::App* parent, const std::string& engine_name);
 
+  void EngineUpdate(CLI::App* parent, const std::string& engine_name);
+
   void EngineGet(CLI::App* parent);
+
+  void EngineUse(CLI::App* parent, const std::string& engine_name);
+
   void ModelUpdate(CLI::App* parent);
 
   CLI::App app_;
@@ -50,6 +55,7 @@ class CommandLineParser {
     bool display_engine = false;
     bool display_version = false;
     std::string filter = "";
+    bool show_menu = false;
 
     int port;
     config_yaml_utils::CortexConfig config;

engine/cli/command_line_parser.h

@@ -29,8 +29,10 @@ struct ChunkParser {
         is_done = true;
       } else {
         try {
-          content = nlohmann::json::parse(s)["choices"][0]["delta"]["content"];
-        } catch (const nlohmann::json::parse_error& e) {
+          content =
+              json_helper::ParseJsonString(s)["choices"][0]["delta"]["content"]
+                  .asString();
+        } catch (const std::exception& e) {
           CTL_WRN("JSON parse error: " << e.what());
         }
       }