Feat/new model folder (#1327)

* feat: pulling interact with new model.list

* feat: return model id when download model success

* feat: model delete for new model.list (#1317)

* feat: models delete for new models data folder structure

* feat: delete model

* feat: cortex chat/run/models start with new model data structure (#1301)

* feat: cortex chat/run/models start with new model data structure

* temp

* fix: use model_id from model service and std::get_line

* f:m

* Model update command/api (#1309)

* Model update command/api

* fix: resume download failed

Signed-off-by: James <[email protected]>

* fix: align github syntax for cuda (#1316)

* fix: require sudo for cortex update (#1318)

* fix: require sudo for cortex update

* fix: comment

* refactor code

* Format code

* Add clean up when finish test

* remove model.list after finish test

* Fix windows CI build

---------

Signed-off-by: James <[email protected]>
Co-authored-by: James <[email protected]>
Co-authored-by: vansangpfiev <[email protected]>

* Add more fields to handle when update

---------

Signed-off-by: James <[email protected]>
Co-authored-by: James <[email protected]>
Co-authored-by: vansangpfiev <[email protected]>

Author: 3 people

engine/commands/chat_cmd.cc

@@ -6,6 +6,7 @@
 #include "server_start_cmd.h"
 #include "trantor/utils/Logger.h"
 #include "utils/logging_utils.h"
+#include "utils/modellist_utils.h"
 
 namespace commands {
 namespace {
@@ -36,23 +37,36 @@ struct ChunkParser {
   }
 };
 
-ChatCmd::ChatCmd(std::string host, int port, const config::ModelConfig& mc)
-    : host_(std::move(host)), port_(port), mc_(mc) {}
+void ChatCmd::Exec(const std::string& host, int port,
+                   const std::string& model_handle, std::string msg) {
+  modellist_utils::ModelListUtils modellist_handler;
+  config::YamlHandler yaml_handler;
+  try {
+    auto model_entry = modellist_handler.GetModelInfo(model_handle);
+    yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml);
+    auto mc = yaml_handler.GetModelConfig();
+    Exec(host, port, mc, std::move(msg));
+  } catch (const std::exception& e) {
+    CLI_LOG("Fail to start model information with ID '" + model_handle +
+            "': " + e.what());
+  }
+}
 
-void ChatCmd::Exec(std::string msg) {
+void ChatCmd::Exec(const std::string& host, int port,
+                   const config::ModelConfig& mc, std::string msg) {
+  auto address = host + ":" + std::to_string(port);
   // Check if server is started
   {
-    if (!commands::IsServerAlive(host_, port_)) {
+    if (!commands::IsServerAlive(host, port)) {
       CLI_LOG("Server is not started yet, please run `"
               << commands::GetCortexBinary() << " start` to start server!");
       return;
     }
   }
 
-  auto address = host_ + ":" + std::to_string(port_);
   // Only check if llamacpp engine
-  if ((mc_.engine.find("llamacpp") != std::string::npos) &&
-      !commands::ModelStatusCmd().IsLoaded(host_, port_, mc_)) {
+  if ((mc.engine.find("llamacpp") != std::string::npos) &&
+      !commands::ModelStatusCmd().IsLoaded(host, port, mc)) {
     CLI_LOG("Model is not loaded yet!");
     return;
   }
@@ -78,12 +92,12 @@ void ChatCmd::Exec(std::string msg) {
         new_data["role"] = kUser;
         new_data["content"] = user_input;
         histories_.push_back(std::move(new_data));
-        json_data["engine"] = mc_.engine;
+        json_data["engine"] = mc.engine;
         json_data["messages"] = histories_;
-        json_data["model"] = mc_.name;
+        json_data["model"] = mc.name;
         //TODO: support non-stream
         json_data["stream"] = true;
-        json_data["stop"] = mc_.stop;
+        json_data["stop"] = mc.stop;
         auto data_str = json_data.dump();
         // std::cout << data_str << std::endl;
         cli.set_read_timeout(std::chrono::seconds(60));

engine/commands/chat_cmd.h

@@ -7,13 +7,12 @@
 namespace commands {
 class ChatCmd {
  public:
-  ChatCmd(std::string host, int port, const config::ModelConfig& mc);
-  void Exec(std::string msg);
+  void Exec(const std::string& host, int port, const std::string& model_handle,
+            std::string msg);
+  void Exec(const std::string& host, int port, const config::ModelConfig& mc,
+            std::string msg);
 
  private:
-  std::string host_;
-  int port_;
-  const config::ModelConfig& mc_;
   std::vector<nlohmann::json> histories_;
 };
 }  // namespace commands

engine/commands/model_del_cmd.cc

@@ -2,55 +2,47 @@
 #include "cmd_info.h"
 #include "config/yaml_config.h"
 #include "utils/file_manager_utils.h"
+#include "utils/modellist_utils.h"
 
 namespace commands {
-bool ModelDelCmd::Exec(const std::string& model_id) {
-  // TODO this implentation may be changed after we have a decision
-  // on https://github.com/janhq/cortex.cpp/issues/1154 but the logic should be similar
-  CmdInfo ci(model_id);
-  std::string model_file =
-      ci.branch == "main" ? ci.model_name : ci.model_name + "-" + ci.branch;
-  auto models_path = file_manager_utils::GetModelsContainerPath();
-  if (std::filesystem::exists(models_path) &&
-      std::filesystem::is_directory(models_path)) {
-    // Iterate through directory
-    for (const auto& entry : std::filesystem::directory_iterator(models_path)) {
-      if (entry.is_regular_file() && entry.path().extension() == ".yaml") {
-        try {
-          config::YamlHandler handler;
-          handler.ModelConfigFromFile(entry.path().string());
-          auto cfg = handler.GetModelConfig();
-          if (entry.path().stem().string() == model_file) {
-            // Delete data
-            if (cfg.files.size() > 0) {
-              std::filesystem::path f(cfg.files[0]);
-              auto rel = std::filesystem::relative(f, models_path);
-              // Only delete model data if it is stored in our models folder
-              if (!rel.empty()) {
-                if (cfg.engine == "cortex.llamacpp") {
-                  std::filesystem::remove_all(f.parent_path());
-                } else {
-                  std::filesystem::remove_all(f);
-                }
-              }
-            }
+bool ModelDelCmd::Exec(const std::string& model_handle) {
+  modellist_utils::ModelListUtils modellist_handler;
+  config::YamlHandler yaml_handler;
 
-            // Delete yaml file
-            std::filesystem::remove(entry);
-            CLI_LOG("The model " << model_id << " was deleted");
-            return true;
+  try {
+    auto model_entry = modellist_handler.GetModelInfo(model_handle);
+    yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml);
+    auto mc = yaml_handler.GetModelConfig();
+    // Remove yaml file
+    std::filesystem::remove(model_entry.path_to_model_yaml);
+    // Remove model files if they are not imported locally
+    if (model_entry.branch_name != "imported") {
+      if (mc.files.size() > 0) {
+        if (mc.engine == "cortex.llamacpp") {
+          for (auto& file : mc.files) {
+            std::filesystem::path gguf_p(file);
+            std::filesystem::remove(gguf_p);
           }
-        } catch (const std::exception& e) {
-          CTL_WRN("Error reading yaml file '" << entry.path().string()
-                                              << "': " << e.what());
-          return false;
+        } else {
+          std::filesystem::path f(mc.files[0]);
+          std::filesystem::remove_all(f);
         }
+      } else {
+        CTL_WRN("model config files are empty!");
       }
     }
-  }
-
-  CLI_LOG("Model does not exist: " << model_id);
 
-  return false;
+    // update model.list
+    if (modellist_handler.DeleteModelEntry(model_handle)) {
+      CLI_LOG("The model " << model_handle << " was deleted");
+      return true;
+    } else {
+      CTL_ERR("Could not delete model: " << model_handle);
+      return false;
+    }
+  } catch (const std::exception& e) {
+    CLI_LOG("Fail to delete model with ID '" + model_handle + "': " + e.what());
+    false;
+  }
 }
 }  // namespace commands

engine/commands/model_del_cmd.h

@@ -6,6 +6,6 @@ namespace commands {
 
 class ModelDelCmd {
  public:
-  bool Exec(const std::string& model_id);
+  bool Exec(const std::string& model_handle);
 };
 }

engine/commands/model_import_cmd.cc

@@ -1,10 +1,8 @@
 #include "model_import_cmd.h"
 #include <filesystem>
-#include <iostream>
 #include <vector>
 #include "config/gguf_parser.h"
 #include "config/yaml_config.h"
-#include "trantor/utils/Logger.h"
 #include "utils/file_manager_utils.h"
 #include "utils/logging_utils.h"
 #include "utils/modellist_utils.h"
@@ -45,7 +43,7 @@ void ModelImportCmd::Exec() {
     }
 
   } catch (const std::exception& e) {
-    // don't need to remove yml file here, because it's written only if model entry is successfully added, 
+    // don't need to remove yml file here, because it's written only if model entry is successfully added,
     // remove file here can make it fail with edge case when user try to import new model with existed model_id
     CLI_LOG("Error importing model path '" + model_path_ + "' with model_id '" +
             model_handle_ + "': " + e.what());

engine/commands/model_start_cmd.cc

@@ -7,43 +7,59 @@
 #include "trantor/utils/Logger.h"
 #include "utils/file_manager_utils.h"
 #include "utils/logging_utils.h"
+#include "utils/modellist_utils.h"
 
 namespace commands {
-ModelStartCmd::ModelStartCmd(std::string host, int port,
-                             const config::ModelConfig& mc)
-    : host_(std::move(host)), port_(port), mc_(mc) {}
+bool ModelStartCmd::Exec(const std::string& host, int port,
+                         const std::string& model_handle) {
 
-bool ModelStartCmd::Exec() {
+  modellist_utils::ModelListUtils modellist_handler;
+  config::YamlHandler yaml_handler;
+  try {
+    auto model_entry = modellist_handler.GetModelInfo(model_handle);
+    yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml);
+    auto mc = yaml_handler.GetModelConfig();
+    return Exec(host, port, mc);
+  } catch (const std::exception& e) {
+    CLI_LOG("Fail to start model information with ID '" + model_handle +
+            "': " + e.what());
+    return false;
+  }
+}
+
+bool ModelStartCmd::Exec(const std::string& host, int port,
+                         const config::ModelConfig& mc) {
   // Check if server is started
-  if (!commands::IsServerAlive(host_, port_)) {
+  if (!commands::IsServerAlive(host, port)) {
     CLI_LOG("Server is not started yet, please run `"
             << commands::GetCortexBinary() << " start` to start server!");
     return false;
   }
+
   // Only check for llamacpp for now
-  if ((mc_.engine.find("llamacpp") != std::string::npos) &&
-      commands::ModelStatusCmd().IsLoaded(host_, port_, mc_)) {
+  if ((mc.engine.find("llamacpp") != std::string::npos) &&
+      commands::ModelStatusCmd().IsLoaded(host, port, mc)) {
     CLI_LOG("Model has already been started!");
     return true;
   }
 
-  httplib::Client cli(host_ + ":" + std::to_string(port_));
+  httplib::Client cli(host + ":" + std::to_string(port));
 
   nlohmann::json json_data;
-  if (mc_.files.size() > 0) {
+  if (mc.files.size() > 0) {
     // TODO(sang) support multiple files
-    json_data["model_path"] = mc_.files[0];
+    json_data["model_path"] = mc.files[0];
   } else {
     LOG_WARN << "model_path is empty";
     return false;
   }
-  json_data["model"] = mc_.name;
-  json_data["system_prompt"] = mc_.system_template;
-  json_data["user_prompt"] = mc_.user_template;
-  json_data["ai_prompt"] = mc_.ai_template;
-  json_data["ctx_len"] = mc_.ctx_len;
-  json_data["stop"] = mc_.stop;
-  json_data["engine"] = mc_.engine;
+  json_data["model"] = mc.name;
+  json_data["system_prompt"] = mc.system_template;
+  json_data["user_prompt"] = mc.user_template;
+  json_data["ai_prompt"] = mc.ai_template;
+  json_data["ctx_len"] = mc.ctx_len;
+  json_data["stop"] = mc.stop;
+  json_data["engine"] = mc.engine;
 
   auto data_str = json_data.dump();
   cli.set_read_timeout(std::chrono::seconds(60));
@@ -52,13 +68,17 @@ bool ModelStartCmd::Exec() {
   if (res) {
     if (res->status == httplib::StatusCode::OK_200) {
       CLI_LOG("Model loaded!");
+      return true;
+    } else {
+      CTL_ERR("Model failed to load with status code: " << res->status);
+      return false;
     }
   } else {
     auto err = res.error();
     CTL_ERR("HTTP error: " << httplib::to_string(err));
     return false;
   }
-  return true;
+  return false;
 }
 
 };  // namespace commands

engine/commands/model_start_cmd.h

@@ -6,13 +6,8 @@ namespace commands {
 
 class ModelStartCmd {
  public:
-  explicit ModelStartCmd(std::string host, int port,
-                         const config::ModelConfig& mc);
-  bool Exec();
+  bool Exec(const std::string& host, int port, const std::string& model_handle);
 
- private:
-  std::string host_;
-  int port_;
-  const config::ModelConfig& mc_;
+  bool Exec(const std::string& host, int port, const config::ModelConfig& mc);
 };
 }  // namespace commands

engine/commands/model_status_cmd.cc

@@ -3,8 +3,25 @@
 #include "httplib.h"
 #include "nlohmann/json.hpp"
 #include "utils/logging_utils.h"
+#include "utils/modellist_utils.h"
 
 namespace commands {
+bool ModelStatusCmd::IsLoaded(const std::string& host, int port,
+                              const std::string& model_handle) {
+  modellist_utils::ModelListUtils modellist_handler;
+  config::YamlHandler yaml_handler;
+  try {
+    auto model_entry = modellist_handler.GetModelInfo(model_handle);
+    yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml);
+    auto mc = yaml_handler.GetModelConfig();
+    return IsLoaded(host, port, mc);
+  } catch (const std::exception& e) {
+    CLI_LOG("Fail to get model status with ID '" + model_handle +
+            "': " + e.what());
+    return false;
+  }
+}
+
 bool ModelStatusCmd::IsLoaded(const std::string& host, int port,
                               const config::ModelConfig& mc) {
   httplib::Client cli(host + ":" + std::to_string(port));

engine/commands/model_status_cmd.h

@@ -6,6 +6,8 @@ namespace commands {
 
 class ModelStatusCmd {
  public:
+  bool IsLoaded(const std::string& host, int port,
+                const std::string& model_handle);
   bool IsLoaded(const std::string& host, int port,
                 const config::ModelConfig& mc);
 };