From 917d45992493b2b95ed6853c042c698767f0c740 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Mon, 23 Sep 2024 15:51:49 +0700 Subject: [PATCH 1/9] Model update command/api --- engine/commands/model_upd_cmd.cc | 224 ++++++++++++++++++++++ engine/commands/model_upd_cmd.h | 64 +++++++ engine/config/model_config.h | 108 +++++++++++ engine/controllers/command_line_parser.cc | 108 ++++++++++- engine/controllers/command_line_parser.h | 14 +- engine/controllers/models.cc | 44 ++++- engine/controllers/models.h | 8 +- 7 files changed, 557 insertions(+), 13 deletions(-) create mode 100644 engine/commands/model_upd_cmd.cc create mode 100644 engine/commands/model_upd_cmd.h diff --git a/engine/commands/model_upd_cmd.cc b/engine/commands/model_upd_cmd.cc new file mode 100644 index 000000000..b1cfdd8e1 --- /dev/null +++ b/engine/commands/model_upd_cmd.cc @@ -0,0 +1,224 @@ +#include "model_upd_cmd.h" +#include "config/yaml_config.h" +#include "utils/logging_utils.h" +#include "utils/modellist_utils.h" +namespace commands { + +ModelUpdCmd::ModelUpdCmd(std::string model_handle) + : model_handle_(std::move(model_handle)) {} + +void ModelUpdCmd::Exec(const ModelUpdateOptions& options) { + modellist_utils::ModelListUtils model_list_utils; + try { + auto model_entry = model_list_utils.GetModelInfo(model_handle_); + config::YamlHandler yaml_handler; + yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml); + config::ModelConfig model_config = yaml_handler.GetModelConfig(); + + // Update only the fields that were passed as arguments + if (!options.name.empty()) { + model_config.name = options.name; + CLI_LOG("Updated name to: " << options.name); + } + if (!options.model.empty()) { + model_config.model = options.model; + CLI_LOG("Updated model to: " << options.model); + } + if (!options.version.empty()) { + model_config.version = options.version; + CLI_LOG("Updated version to: " << options.version); + } + if (!options.stop.empty()) { + std::istringstream iss(options.stop); + std::string token; + model_config.stop.clear(); + while (std::getline(iss, token, ',')) { + model_config.stop.push_back(token); + } + CLI_LOG("Updated stop to: " << options.stop); + } + if (!options.top_p.empty()) { + model_config.top_p = std::stof(options.top_p); + CLI_LOG("Updated top_p to: " << options.top_p); + } + if (!options.temperature.empty()) { + model_config.temperature = std::stof(options.temperature); + CLI_LOG("Updated temperature to: " << options.temperature); + } + if (!options.frequency_penalty.empty()) { + model_config.frequency_penalty = std::stof(options.frequency_penalty); + CLI_LOG("Updated frequency_penalty to: " << options.frequency_penalty); + } + if (!options.presence_penalty.empty()) { + model_config.presence_penalty = std::stof(options.presence_penalty); + CLI_LOG("Updated presence_penalty to: " << options.presence_penalty); + } + if (!options.max_tokens.empty()) { + model_config.max_tokens = std::stoi(options.max_tokens); + CLI_LOG("Updated max_tokens to: " << options.max_tokens); + } + if (!options.stream.empty()) { + model_config.stream = (options.stream == "true" || options.stream == "1"); + CLI_LOG("Updated stream to: " << options.stream); + } + if (!options.ngl.empty()) { + model_config.ngl = std::stoi(options.ngl); + CLI_LOG("Updated ngl to: " << options.ngl); + } + if (!options.ctx_len.empty()) { + model_config.ctx_len = std::stoi(options.ctx_len); + CLI_LOG("Updated ctx_len to: " << options.ctx_len); + } + if (!options.engine.empty()) { + model_config.engine = options.engine; + CLI_LOG("Updated engine to: " << options.engine); + } + if (!options.prompt_template.empty()) { + model_config.prompt_template = options.prompt_template; + CLI_LOG("Updated prompt_template to: " << options.prompt_template); + } + if (!options.system_template.empty()) { + model_config.system_template = options.system_template; + CLI_LOG("Updated system_template to: " << options.system_template); + } + if (!options.user_template.empty()) { + model_config.user_template = options.user_template; + CLI_LOG("Updated user_template to: " << options.user_template); + } + if (!options.ai_template.empty()) { + model_config.ai_template = options.ai_template; + CLI_LOG("Updated ai_template to: " << options.ai_template); + } + if (!options.os.empty()) { + model_config.os = options.os; + CLI_LOG("Updated os to: " << options.os); + } + if (!options.gpu_arch.empty()) { + model_config.gpu_arch = options.gpu_arch; + CLI_LOG("Updated gpu_arch to: " << options.gpu_arch); + } + if (!options.quantization_method.empty()) { + model_config.quantization_method = options.quantization_method; + CLI_LOG( + "Updated quantization_method to: " << options.quantization_method); + } + if (!options.precision.empty()) { + model_config.precision = options.precision; + CLI_LOG("Updated precision to: " << options.precision); + } + if (!options.tp.empty()) { + model_config.tp = std::stoi(options.tp); + CLI_LOG("Updated tp to: " << options.tp); + } + if (!options.trtllm_version.empty()) { + model_config.trtllm_version = options.trtllm_version; + CLI_LOG("Updated trtllm_version to: " << options.trtllm_version); + } + if (!options.text_model.empty()) { + model_config.text_model = + (options.text_model == "true" || options.text_model == "1"); + CLI_LOG("Updated text_model to: " << options.text_model); + } + if (!options.files.empty()) { + std::istringstream iss(options.files); + std::string token; + model_config.files.clear(); + while (std::getline(iss, token, ',')) { + model_config.files.push_back(token); + } + CLI_LOG("Updated files to: " << options.files); + } + if (!options.created.empty()) { + model_config.created = std::stoull(options.created); + CLI_LOG("Updated created to: " << options.created); + } + if (!options.object.empty()) { + model_config.object = options.object; + CLI_LOG("Updated object to: " << options.object); + } + if (!options.owned_by.empty()) { + model_config.owned_by = options.owned_by; + CLI_LOG("Updated owned_by to: " << options.owned_by); + } + if (!options.seed.empty()) { + model_config.seed = std::stoi(options.seed); + CLI_LOG("Updated seed to: " << options.seed); + } + if (!options.dynatemp_range.empty()) { + model_config.dynatemp_range = std::stof(options.dynatemp_range); + CLI_LOG("Updated dynatemp_range to: " << options.dynatemp_range); + } + if (!options.dynatemp_exponent.empty()) { + model_config.dynatemp_exponent = std::stof(options.dynatemp_exponent); + CLI_LOG("Updated dynatemp_exponent to: " << options.dynatemp_exponent); + } + if (!options.top_k.empty()) { + model_config.top_k = std::stoi(options.top_k); + CLI_LOG("Updated top_k to: " << options.top_k); + } + if (!options.min_p.empty()) { + model_config.min_p = std::stof(options.min_p); + CLI_LOG("Updated min_p to: " << options.min_p); + } + if (!options.tfs_z.empty()) { + model_config.tfs_z = std::stof(options.tfs_z); + CLI_LOG("Updated tfs_z to: " << options.tfs_z); + } + if (!options.typ_p.empty()) { + model_config.typ_p = std::stof(options.typ_p); + CLI_LOG("Updated typ_p to: " << options.typ_p); + } + if (!options.repeat_last_n.empty()) { + model_config.repeat_last_n = std::stoi(options.repeat_last_n); + CLI_LOG("Updated repeat_last_n to: " << options.repeat_last_n); + } + if (!options.repeat_penalty.empty()) { + model_config.repeat_penalty = std::stof(options.repeat_penalty); + CLI_LOG("Updated repeat_penalty to: " << options.repeat_penalty); + } + if (!options.mirostat.empty()) { + model_config.mirostat = + (options.mirostat == "true" || options.mirostat == "1"); + CLI_LOG("Updated mirostat to: " << options.mirostat); + } + if (!options.mirostat_tau.empty()) { + model_config.mirostat_tau = std::stof(options.mirostat_tau); + CLI_LOG("Updated mirostat_tau to: " << options.mirostat_tau); + } + if (!options.mirostat_eta.empty()) { + model_config.mirostat_eta = std::stof(options.mirostat_eta); + CLI_LOG("Updated mirostat_eta to: " << options.mirostat_eta); + } + if (!options.penalize_nl.empty()) { + model_config.penalize_nl = + (options.penalize_nl == "true" || options.penalize_nl == "1"); + CLI_LOG("Updated penalize_nl to: " << options.penalize_nl); + } + if (!options.ignore_eos.empty()) { + model_config.ignore_eos = + (options.ignore_eos == "true" || options.ignore_eos == "1"); + CLI_LOG("Updated ignore_eos to: " << options.ignore_eos); + } + if (!options.n_probs.empty()) { + model_config.n_probs = std::stoi(options.n_probs); + CLI_LOG("Updated n_probs to: " << options.n_probs); + } + if (!options.min_keep.empty()) { + model_config.min_keep = std::stoi(options.min_keep); + CLI_LOG("Updated min_keep to: " << options.min_keep); + } + if (!options.grammar.empty()) { + model_config.grammar = options.grammar; + CLI_LOG("Updated grammar to: " << options.grammar); + } + yaml_handler.UpdateModelConfig(model_config); + + yaml_handler.WriteYamlFile(model_entry.path_to_model_yaml); + CLI_LOG("Successfully update model ID '" + model_handle_ + "'!"); + } catch (const std::exception& e) { + CLI_LOG("Fail to update model with model ID '" + model_handle_ + + "': " + e.what()); + } +} + +} // namespace commands \ No newline at end of file diff --git a/engine/commands/model_upd_cmd.h b/engine/commands/model_upd_cmd.h new file mode 100644 index 000000000..0e94767aa --- /dev/null +++ b/engine/commands/model_upd_cmd.h @@ -0,0 +1,64 @@ +#pragma once +#include +#include +#include +#include +#include +#include "config/model_config.h" +namespace commands { +struct ModelUpdateOptions { + std::string name; + std::string model; + std::string version; + std::string stop; + std::string top_p; + std::string temperature; + std::string frequency_penalty; + std::string presence_penalty; + std::string max_tokens; + std::string stream; + std::string ngl; + std::string ctx_len; + std::string engine; + std::string prompt_template; + std::string system_template; + std::string user_template; + std::string ai_template; + std::string os; + std::string gpu_arch; + std::string quantization_method; + std::string precision; + std::string tp; + std::string trtllm_version; + std::string text_model; + std::string files; + std::string created; + std::string object; + std::string owned_by; + std::string seed; + std::string dynatemp_range; + std::string dynatemp_exponent; + std::string top_k; + std::string min_p; + std::string tfs_z; + std::string typ_p; + std::string repeat_last_n; + std::string repeat_penalty; + std::string mirostat; + std::string mirostat_tau; + std::string mirostat_eta; + std::string penalize_nl; + std::string ignore_eos; + std::string n_probs; + std::string min_keep; + std::string grammar; +}; +class ModelUpdCmd { + public: + ModelUpdCmd(std::string model_handle); + void Exec(const ModelUpdateOptions& options); + + private: + std::string model_handle_; +}; +} // namespace commands \ No newline at end of file diff --git a/engine/config/model_config.h b/engine/config/model_config.h index 74410db52..a65114ca7 100644 --- a/engine/config/model_config.h +++ b/engine/config/model_config.h @@ -58,7 +58,115 @@ struct ModelConfig { int n_probs = 0; int min_keep = 0; std::string grammar; + + void FromJson(const Json::Value& json) { + // do now allow to update ID and model field because it is unique identifier + // if (json.isMember("id")) + // id = json["id"].asString(); + if (json.isMember("name")) + name = json["name"].asString(); + // if (json.isMember("model")) + // model = json["model"].asString(); + if (json.isMember("version")) + version = json["version"].asString(); + if (json.isMember("stop") && json["stop"].isArray()) { + stop.clear(); + for (const auto& s : json["stop"]) { + stop.push_back(s.asString()); + } + } + + if (json.isMember("stream")) + stream = json["stream"].asBool(); + if (json.isMember("top_p")) + top_p = json["top_p"].asFloat(); + if (json.isMember("temperature")) + temperature = json["temperature"].asFloat(); + if (json.isMember("frequency_penalty")) + frequency_penalty = json["frequency_penalty"].asFloat(); + if (json.isMember("presence_penalty")) + presence_penalty = json["presence_penalty"].asFloat(); + if (json.isMember("max_tokens")) + max_tokens = json["max_tokens"].asInt(); + if (json.isMember("seed")) + seed = json["seed"].asInt(); + if (json.isMember("dynatemp_range")) + dynatemp_range = json["dynatemp_range"].asFloat(); + if (json.isMember("dynatemp_exponent")) + dynatemp_exponent = json["dynatemp_exponent"].asFloat(); + if (json.isMember("top_k")) + top_k = json["top_k"].asInt(); + if (json.isMember("min_p")) + min_p = json["min_p"].asFloat(); + if (json.isMember("tfs_z")) + tfs_z = json["tfs_z"].asFloat(); + if (json.isMember("typ_p")) + typ_p = json["typ_p"].asFloat(); + if (json.isMember("repeat_last_n")) + repeat_last_n = json["repeat_last_n"].asInt(); + if (json.isMember("repeat_penalty")) + repeat_penalty = json["repeat_penalty"].asFloat(); + if (json.isMember("mirostat")) + mirostat = json["mirostat"].asBool(); + if (json.isMember("mirostat_tau")) + mirostat_tau = json["mirostat_tau"].asFloat(); + if (json.isMember("mirostat_eta")) + mirostat_eta = json["mirostat_eta"].asFloat(); + if (json.isMember("penalize_nl")) + penalize_nl = json["penalize_nl"].asBool(); + if (json.isMember("ignore_eos")) + ignore_eos = json["ignore_eos"].asBool(); + if (json.isMember("n_probs")) + n_probs = json["n_probs"].asInt(); + if (json.isMember("min_keep")) + min_keep = json["min_keep"].asInt(); + if (json.isMember("ngl")) + ngl = json["ngl"].asInt(); + if (json.isMember("ctx_len")) + ctx_len = json["ctx_len"].asInt(); + if (json.isMember("engine")) + engine = json["engine"].asString(); + if (json.isMember("prompt_template")) + prompt_template = json["prompt_template"].asString(); + if (json.isMember("system_template")) + system_template = json["system_template"].asString(); + if (json.isMember("user_template")) + user_template = json["user_template"].asString(); + if (json.isMember("ai_template")) + ai_template = json["ai_template"].asString(); + if (json.isMember("os")) + os = json["os"].asString(); + if (json.isMember("gpu_arch")) + gpu_arch = json["gpu_arch"].asString(); + if (json.isMember("quantization_method")) + quantization_method = json["quantization_method"].asString(); + if (json.isMember("precision")) + precision = json["precision"].asString(); + + if (json.isMember("files") && json["files"].isArray()) { + files.clear(); + for (const auto& file : json["files"]) { + files.push_back(file.asString()); + } + } + + if (json.isMember("created")) + created = json["created"].asUInt64(); + if (json.isMember("object")) + object = json["object"].asString(); + if (json.isMember("owned_by")) + owned_by = json["owned_by"].asString(); + if (json.isMember("text_model")) + text_model = json["text_model"].asBool(); + + if (engine == "cortex.tensorrt-llm") { + if (json.isMember("trtllm_version")) + trtllm_version = json["trtllm_version"].asString(); + if (json.isMember("tp")) + tp = json["tp"].asInt(); + } + } Json::Value ToJson() const { Json::Value obj; diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index f57efb7a2..760c7e25c 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -14,6 +14,7 @@ #include "commands/model_pull_cmd.h" #include "commands/model_start_cmd.h" #include "commands/model_stop_cmd.h" +#include "commands/model_upd_cmd.h" #include "commands/run_cmd.h" #include "commands/server_start_cmd.h" #include "commands/server_stop_cmd.h" @@ -271,10 +272,8 @@ void CommandLineParser::SetupModelCommands() { commands::ModelAliasCmd mdc; mdc.Exec(cml_data_.model_id, cml_data_.model_alias); }); - - auto model_update_cmd = - models_cmd->add_subcommand("update", "Update configuration of a model"); - model_update_cmd->group(kSubcommands); + // Model update parameters comment + ModelUpdate(models_cmd); std::string model_path; auto model_import_cmd = models_cmd->add_subcommand( @@ -453,3 +452,104 @@ void CommandLineParser::EngineGet(CLI::App* parent) { [engine_name] { commands::EngineGetCmd().Exec(engine_name); }); } } + +void CommandLineParser::ModelUpdate(CLI::App* parent) { + auto model_update_cmd = + parent->add_subcommand("update", "Update configuration of a model"); + model_update_cmd->group(kSubcommands); + model_update_cmd->add_option("--model_id", cml_data_.model_id, "Model ID") + ->required(); + + // Add options dynamically + model_update_cmd->add_option("--name", model_update_options_.name, "Name"); + model_update_cmd->add_option("--model", model_update_options_.model, "Model"); + model_update_cmd->add_option("--version", model_update_options_.version, + "Version"); + model_update_cmd->add_option("--stop", model_update_options_.stop, "Stop"); + model_update_cmd->add_option("--top_p", model_update_options_.top_p, "Top P"); + model_update_cmd->add_option( + "--temperature", model_update_options_.temperature, "Temperature"); + model_update_cmd->add_option("--frequency_penalty", + model_update_options_.frequency_penalty, + "Frequency Penalty"); + model_update_cmd->add_option("--presence_penalty", + model_update_options_.presence_penalty, + "Presence Penalty"); + model_update_cmd->add_option("--max_tokens", model_update_options_.max_tokens, + "Max Tokens"); + model_update_cmd->add_option("--stream", model_update_options_.stream, + "Stream"); + model_update_cmd->add_option("--ngl", model_update_options_.ngl, "NGL"); + model_update_cmd->add_option("--ctx_len", model_update_options_.ctx_len, + "Context Length"); + model_update_cmd->add_option("--engine", model_update_options_.engine, + "Engine"); + model_update_cmd->add_option("--prompt_template", + model_update_options_.prompt_template, + "Prompt Template"); + model_update_cmd->add_option("--system_template", + model_update_options_.system_template, + "System Template"); + model_update_cmd->add_option( + "--user_template", model_update_options_.user_template, "User Template"); + model_update_cmd->add_option( + "--ai_template", model_update_options_.ai_template, "AI Template"); + model_update_cmd->add_option("--os", model_update_options_.os, "OS"); + model_update_cmd->add_option("--gpu_arch", model_update_options_.gpu_arch, + "GPU Architecture"); + model_update_cmd->add_option("--quantization_method", + model_update_options_.quantization_method, + "Quantization Method"); + model_update_cmd->add_option("--precision", model_update_options_.precision, + "Precision"); + model_update_cmd->add_option("--tp", model_update_options_.tp, "TP"); + model_update_cmd->add_option("--trtllm_version", + model_update_options_.trtllm_version, + "TRTLLM Version"); + model_update_cmd->add_option("--text_model", model_update_options_.text_model, + "Text Model"); + model_update_cmd->add_option("--files", model_update_options_.files, "Files"); + model_update_cmd->add_option("--created", model_update_options_.created, + "Created"); + model_update_cmd->add_option("--object", model_update_options_.object, + "Object"); + model_update_cmd->add_option("--owned_by", model_update_options_.owned_by, + "Owned By"); + model_update_cmd->add_option("--seed", model_update_options_.seed, "Seed"); + model_update_cmd->add_option("--dynatemp_range", + model_update_options_.dynatemp_range, + "Dynatemp Range"); + model_update_cmd->add_option("--dynatemp_exponent", + model_update_options_.dynatemp_exponent, + "Dynatemp Exponent"); + model_update_cmd->add_option("--top_k", model_update_options_.top_k, "Top K"); + model_update_cmd->add_option("--min_p", model_update_options_.min_p, "Min P"); + model_update_cmd->add_option("--tfs_z", model_update_options_.tfs_z, "TFS Z"); + model_update_cmd->add_option("--typ_p", model_update_options_.typ_p, "Typ P"); + model_update_cmd->add_option( + "--repeat_last_n", model_update_options_.repeat_last_n, "Repeat Last N"); + model_update_cmd->add_option("--repeat_penalty", + model_update_options_.repeat_penalty, + "Repeat Penalty"); + model_update_cmd->add_option("--mirostat", model_update_options_.mirostat, + "Mirostat"); + model_update_cmd->add_option( + "--mirostat_tau", model_update_options_.mirostat_tau, "Mirostat Tau"); + model_update_cmd->add_option( + "--mirostat_eta", model_update_options_.mirostat_eta, "Mirostat Eta"); + model_update_cmd->add_option( + "--penalize_nl", model_update_options_.penalize_nl, "Penalize NL"); + model_update_cmd->add_option("--ignore_eos", model_update_options_.ignore_eos, + "Ignore EOS"); + model_update_cmd->add_option("--n_probs", model_update_options_.n_probs, + "N Probs"); + model_update_cmd->add_option("--min_keep", model_update_options_.min_keep, + "Min Keep"); + model_update_cmd->add_option("--grammar", model_update_options_.grammar, + "Grammar"); + + model_update_cmd->callback([this]() { + commands::ModelUpdCmd command(cml_data_.model_id); + command.Exec(model_update_options_); + }); +} \ No newline at end of file diff --git a/engine/controllers/command_line_parser.h b/engine/controllers/command_line_parser.h index 98f437098..25c201188 100644 --- a/engine/controllers/command_line_parser.h +++ b/engine/controllers/command_line_parser.h @@ -1,9 +1,9 @@ #pragma once #include "CLI/CLI.hpp" +#include "commands/model_upd_cmd.h" #include "services/engine_service.h" #include "utils/config_yaml_utils.h" - class CommandLineParser { public: CommandLineParser(); @@ -11,13 +11,13 @@ class CommandLineParser { private: void SetupCommonCommands(); - + void SetupInferenceCommands(); - + void SetupModelCommands(); - + void SetupEngineCommands(); - + void SetupSystemCommands(); void EngineInstall(CLI::App* parent, const std::string& engine_name, @@ -26,10 +26,11 @@ class CommandLineParser { void EngineUninstall(CLI::App* parent, const std::string& engine_name); void EngineGet(CLI::App* parent); + void ModelUpdate(CLI::App* parent); CLI::App app_; EngineService engine_service_; - struct CmlData{ + struct CmlData { std::string model_id; std::string msg; std::string model_alias; @@ -41,4 +42,5 @@ class CommandLineParser { config_yaml_utils::CortexConfig config; }; CmlData cml_data_; + commands::ModelUpdateOptions model_update_options_; }; diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index e857d89da..4660b50e5 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -114,7 +114,7 @@ void Models::GetModel( auto model_config = yaml_handler.GetModelConfig(); Json::Value obj = model_config.ToJson(); - + data.append(std::move(obj)); ret["data"] = data; ret["result"] = "OK"; @@ -155,7 +155,49 @@ void Models::DeleteModel(const HttpRequestPtr& req, callback(resp); } } +void Models::UpdateModel( + const HttpRequestPtr& req, + std::function&& callback) const { + if (!http_util::HasFieldInReq(req, callback, "modelId")) { + return; + } + auto model_id = (*(req->getJsonObject())).get("modelId", "").asString(); + auto json_body = *(req->getJsonObject()); + try { + modellist_utils::ModelListUtils model_list_utils; + auto model_entry = model_list_utils.GetModelInfo(model_id); + config::YamlHandler yaml_handler; + yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml); + config::ModelConfig model_config = yaml_handler.GetModelConfig(); + model_config.FromJson(json_body); + yaml_handler.UpdateModelConfig(model_config); + yaml_handler.WriteYamlFile(model_entry.path_to_model_yaml); + std::string message = "Successfully update model ID '" + model_id + + "': " + json_body.toStyledString(); + LOG_INFO << message; + Json::Value ret; + ret["result"] = "Updated successfully!"; + ret["modelHandle"] = model_id; + ret["message"] = message; + + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k400BadRequest); + callback(resp); + + } catch (const std::exception& e) { + std::string error_message = + "Error updating with model_id '" + model_id + "': " + e.what(); + LOG_ERROR << error_message; + Json::Value ret; + ret["result"] = "Updated failed!"; + ret["modelHandle"] = model_id; + ret["message"] = error_message; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k400BadRequest); + callback(resp); + } +} void Models::ImportModel( const HttpRequestPtr& req, std::function&& callback) const { diff --git a/engine/controllers/models.h b/engine/controllers/models.h index 4ae1ff41f..8d652c86a 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -15,6 +15,7 @@ class Models : public drogon::HttpController { METHOD_ADD(Models::PullModel, "/pull", Post); METHOD_ADD(Models::ListModel, "/list", Get); METHOD_ADD(Models::GetModel, "/get", Post); + METHOD_ADD(Models::UpdateModel, "/update/", Post); METHOD_ADD(Models::ImportModel, "/import", Post); METHOD_ADD(Models::DeleteModel, "/{1}", Delete); METHOD_ADD(Models::SetModelAlias, "/alias", Post); @@ -26,8 +27,11 @@ class Models : public drogon::HttpController { std::function&& callback) const; void GetModel(const HttpRequestPtr& req, std::function&& callback) const; - void ImportModel(const HttpRequestPtr& req, - std::function&& callback) const; + void UpdateModel(const HttpRequestPtr& req, + std::function&& callback) const; + void ImportModel( + const HttpRequestPtr& req, + std::function&& callback) const; void DeleteModel(const HttpRequestPtr& req, std::function&& callback, const std::string& model_id) const; From b88b1bda5c3fc89a81e9b1817f52899280991cfc Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Sep 2024 22:29:30 +0700 Subject: [PATCH 2/9] fix: resume download failed Signed-off-by: James --- engine/services/download_service.cc | 60 ++++++++++++++++++++++++----- engine/services/download_service.h | 3 ++ 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 1cf8b68c4..471a70013 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -12,6 +12,14 @@ #include "utils/format_utils.h" #include "utils/logging_utils.h" +#ifdef _WIN32 +#define ftell64(f) _ftelli64(f) +#define fseek64(f, o, w) _fseeki64(f, o, w) +#else +#define ftell64(f) ftello(f) +#define fseek64(f, o, w) fseeko(f, o, w) +#endif + namespace { size_t WriteCallback(void* ptr, size_t size, size_t nmemb, FILE* stream) { size_t written = fwrite(ptr, size, nmemb, stream); @@ -37,12 +45,19 @@ void DownloadService::AddDownloadTask( } // all items are valid, start downloading + bool download_successfully = true; for (const auto& item : task.items) { CLI_LOG("Start downloading: " + item.localPath.filename().string()); - Download(task.id, item, true); + try { + Download(task.id, item, true); + } catch (const std::runtime_error& e) { + CTL_ERR("Failed to download: " << item.downloadUrl << " - " << e.what()); + download_successfully = false; + break; + } } - if (callback.has_value()) { + if (download_successfully && callback.has_value()) { callback.value()(task); } } @@ -102,10 +117,15 @@ void DownloadService::Download(const std::string& download_id, std::string mode = "wb"; if (allow_resume && std::filesystem::exists(download_item.localPath) && download_item.bytes.has_value()) { - FILE* existing_file = fopen(download_item.localPath.string().c_str(), "r"); - fseek(existing_file, 0, SEEK_END); - curl_off_t existing_file_size = ftell(existing_file); - fclose(existing_file); + curl_off_t existing_file_size = GetLocalFileSize(download_item.localPath); + if (existing_file_size == -1) { + CLI_LOG("Cannot get file size: " << download_item.localPath.string() + << " . Start download over!"); + return; + } + CTL_INF("Existing file size: " << download_item.downloadUrl << " - " + << download_item.localPath.string() << " - " + << existing_file_size); auto missing_bytes = download_item.bytes.value() - existing_file_size; if (missing_bytes > 0) { CLI_LOG("Found unfinished download! Additional " @@ -149,9 +169,13 @@ void DownloadService::Download(const std::string& download_id, curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); if (mode == "ab") { - fseek(file, 0, SEEK_END); - curl_off_t local_file_size = ftell(file); - curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, local_file_size); + auto local_file_size = GetLocalFileSize(download_item.localPath); + if (local_file_size != -1) { + curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, + GetLocalFileSize(download_item.localPath)); + } else { + CTL_ERR("Cannot get file size: " << download_item.localPath.string()); + } } res = curl_easy_perform(curl); @@ -159,8 +183,26 @@ void DownloadService::Download(const std::string& download_id, if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + throw std::runtime_error("Failed to download file " + + download_item.localPath.filename().string()); } fclose(file); curl_easy_cleanup(curl); } + +curl_off_t DownloadService::GetLocalFileSize( + const std::filesystem::path& path) const { + FILE* file = fopen(path.string().c_str(), "r"); + if (!file) { + return -1; + } + + if (fseek64(file, 0, SEEK_END) != 0) { + return -1; + } + + curl_off_t file_size = ftell64(file); + fclose(file); + return file_size; +} \ No newline at end of file diff --git a/engine/services/download_service.h b/engine/services/download_service.h index 7063be74c..b9f93ee82 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -73,4 +74,6 @@ class DownloadService { private: void Download(const std::string& download_id, const DownloadItem& download_item, bool allow_resume); + + curl_off_t GetLocalFileSize(const std::filesystem::path& path) const; }; From 5e7bda4d0087f58810efde0a5d9cf0f1487cc449 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 24 Sep 2024 08:32:37 +0700 Subject: [PATCH 3/9] fix: align github syntax for cuda (#1316) --- engine/services/engine_service.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 1b1f1d278..289bebd68 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -119,9 +119,12 @@ void EngineService::UnzipEngine(const std::string& engine, CTL_INF("engine: " << engine); CTL_INF("CUDA version: " << hw_inf_.cuda_driver_version); std::string cuda_variant = "cuda-"; - cuda_variant += GetSuitableCudaVersion(engine, hw_inf_.cuda_driver_version) + - "-" + hw_inf_.sys_inf->os + "-" + hw_inf_.sys_inf->arch + - ".tar.gz"; + auto cuda_github = + GetSuitableCudaVersion(engine, hw_inf_.cuda_driver_version); + // Github release cuda example: cuda-12-0-windows-amd64.tar.gz + std::replace(cuda_github.begin(), cuda_github.end(), '.', '-'); + cuda_variant += cuda_github + "-" + hw_inf_.sys_inf->os + "-" + + hw_inf_.sys_inf->arch + ".tar.gz"; CTL_INF("cuda_variant: " << cuda_variant); std::vector variants; From 623e870142c85243c0cedb9852a8be8eea9b3e9b Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 24 Sep 2024 09:24:22 +0700 Subject: [PATCH 4/9] fix: require sudo for cortex update (#1318) * fix: require sudo for cortex update * fix: comment --- engine/controllers/command_line_parser.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 31ace9ffd..d64104197 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -388,6 +388,12 @@ void CommandLineParser::SetupSystemCommands() { update_cmd->group(kSystemGroup); update_cmd->add_option("-v", cml_data_.cortex_version, ""); update_cmd->callback([this] { +#if !defined(_WIN32) + if (getuid()) { + CLI_LOG("Error: Not root user. Please run with sudo."); + return; + } +#endif commands::CortexUpdCmd cuc; cuc.Exec(cml_data_.cortex_version); cml_data_.check_upd = false; From c5d00bf97593e449bf85198195d2adcfb8eea3bf Mon Sep 17 00:00:00 2001 From: Thuandz Date: Tue, 24 Sep 2024 10:52:55 +0700 Subject: [PATCH 5/9] refactor code --- engine/commands/model_upd_cmd.cc | 315 ++++++++-------------- engine/commands/model_upd_cmd.h | 62 +---- engine/controllers/command_line_parser.cc | 112 ++------ engine/controllers/command_line_parser.h | 2 +- 4 files changed, 146 insertions(+), 345 deletions(-) diff --git a/engine/commands/model_upd_cmd.cc b/engine/commands/model_upd_cmd.cc index b1cfdd8e1..eb7edd3df 100644 --- a/engine/commands/model_upd_cmd.cc +++ b/engine/commands/model_upd_cmd.cc @@ -1,224 +1,127 @@ #include "model_upd_cmd.h" -#include "config/yaml_config.h" + #include "utils/logging_utils.h" -#include "utils/modellist_utils.h" + namespace commands { ModelUpdCmd::ModelUpdCmd(std::string model_handle) : model_handle_(std::move(model_handle)) {} -void ModelUpdCmd::Exec(const ModelUpdateOptions& options) { - modellist_utils::ModelListUtils model_list_utils; +void ModelUpdCmd::Exec( + const std::unordered_map& options) { try { - auto model_entry = model_list_utils.GetModelInfo(model_handle_); - config::YamlHandler yaml_handler; - yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml); - config::ModelConfig model_config = yaml_handler.GetModelConfig(); + auto model_entry = model_list_utils_.GetModelInfo(model_handle_); + yaml_handler_.ModelConfigFromFile(model_entry.path_to_model_yaml); + model_config_ = yaml_handler_.GetModelConfig(); - // Update only the fields that were passed as arguments - if (!options.name.empty()) { - model_config.name = options.name; - CLI_LOG("Updated name to: " << options.name); - } - if (!options.model.empty()) { - model_config.model = options.model; - CLI_LOG("Updated model to: " << options.model); - } - if (!options.version.empty()) { - model_config.version = options.version; - CLI_LOG("Updated version to: " << options.version); - } - if (!options.stop.empty()) { - std::istringstream iss(options.stop); - std::string token; - model_config.stop.clear(); - while (std::getline(iss, token, ',')) { - model_config.stop.push_back(token); - } - CLI_LOG("Updated stop to: " << options.stop); - } - if (!options.top_p.empty()) { - model_config.top_p = std::stof(options.top_p); - CLI_LOG("Updated top_p to: " << options.top_p); - } - if (!options.temperature.empty()) { - model_config.temperature = std::stof(options.temperature); - CLI_LOG("Updated temperature to: " << options.temperature); - } - if (!options.frequency_penalty.empty()) { - model_config.frequency_penalty = std::stof(options.frequency_penalty); - CLI_LOG("Updated frequency_penalty to: " << options.frequency_penalty); - } - if (!options.presence_penalty.empty()) { - model_config.presence_penalty = std::stof(options.presence_penalty); - CLI_LOG("Updated presence_penalty to: " << options.presence_penalty); - } - if (!options.max_tokens.empty()) { - model_config.max_tokens = std::stoi(options.max_tokens); - CLI_LOG("Updated max_tokens to: " << options.max_tokens); - } - if (!options.stream.empty()) { - model_config.stream = (options.stream == "true" || options.stream == "1"); - CLI_LOG("Updated stream to: " << options.stream); - } - if (!options.ngl.empty()) { - model_config.ngl = std::stoi(options.ngl); - CLI_LOG("Updated ngl to: " << options.ngl); - } - if (!options.ctx_len.empty()) { - model_config.ctx_len = std::stoi(options.ctx_len); - CLI_LOG("Updated ctx_len to: " << options.ctx_len); - } - if (!options.engine.empty()) { - model_config.engine = options.engine; - CLI_LOG("Updated engine to: " << options.engine); - } - if (!options.prompt_template.empty()) { - model_config.prompt_template = options.prompt_template; - CLI_LOG("Updated prompt_template to: " << options.prompt_template); - } - if (!options.system_template.empty()) { - model_config.system_template = options.system_template; - CLI_LOG("Updated system_template to: " << options.system_template); - } - if (!options.user_template.empty()) { - model_config.user_template = options.user_template; - CLI_LOG("Updated user_template to: " << options.user_template); - } - if (!options.ai_template.empty()) { - model_config.ai_template = options.ai_template; - CLI_LOG("Updated ai_template to: " << options.ai_template); - } - if (!options.os.empty()) { - model_config.os = options.os; - CLI_LOG("Updated os to: " << options.os); - } - if (!options.gpu_arch.empty()) { - model_config.gpu_arch = options.gpu_arch; - CLI_LOG("Updated gpu_arch to: " << options.gpu_arch); - } - if (!options.quantization_method.empty()) { - model_config.quantization_method = options.quantization_method; - CLI_LOG( - "Updated quantization_method to: " << options.quantization_method); - } - if (!options.precision.empty()) { - model_config.precision = options.precision; - CLI_LOG("Updated precision to: " << options.precision); - } - if (!options.tp.empty()) { - model_config.tp = std::stoi(options.tp); - CLI_LOG("Updated tp to: " << options.tp); - } - if (!options.trtllm_version.empty()) { - model_config.trtllm_version = options.trtllm_version; - CLI_LOG("Updated trtllm_version to: " << options.trtllm_version); - } - if (!options.text_model.empty()) { - model_config.text_model = - (options.text_model == "true" || options.text_model == "1"); - CLI_LOG("Updated text_model to: " << options.text_model); - } - if (!options.files.empty()) { - std::istringstream iss(options.files); - std::string token; - model_config.files.clear(); - while (std::getline(iss, token, ',')) { - model_config.files.push_back(token); + for (const auto& [key, value] : options) { + if (!value.empty()) { + UpdateConfig(key, value); } - CLI_LOG("Updated files to: " << options.files); - } - if (!options.created.empty()) { - model_config.created = std::stoull(options.created); - CLI_LOG("Updated created to: " << options.created); - } - if (!options.object.empty()) { - model_config.object = options.object; - CLI_LOG("Updated object to: " << options.object); - } - if (!options.owned_by.empty()) { - model_config.owned_by = options.owned_by; - CLI_LOG("Updated owned_by to: " << options.owned_by); - } - if (!options.seed.empty()) { - model_config.seed = std::stoi(options.seed); - CLI_LOG("Updated seed to: " << options.seed); - } - if (!options.dynatemp_range.empty()) { - model_config.dynatemp_range = std::stof(options.dynatemp_range); - CLI_LOG("Updated dynatemp_range to: " << options.dynatemp_range); } - if (!options.dynatemp_exponent.empty()) { - model_config.dynatemp_exponent = std::stof(options.dynatemp_exponent); - CLI_LOG("Updated dynatemp_exponent to: " << options.dynatemp_exponent); - } - if (!options.top_k.empty()) { - model_config.top_k = std::stoi(options.top_k); - CLI_LOG("Updated top_k to: " << options.top_k); - } - if (!options.min_p.empty()) { - model_config.min_p = std::stof(options.min_p); - CLI_LOG("Updated min_p to: " << options.min_p); - } - if (!options.tfs_z.empty()) { - model_config.tfs_z = std::stof(options.tfs_z); - CLI_LOG("Updated tfs_z to: " << options.tfs_z); - } - if (!options.typ_p.empty()) { - model_config.typ_p = std::stof(options.typ_p); - CLI_LOG("Updated typ_p to: " << options.typ_p); - } - if (!options.repeat_last_n.empty()) { - model_config.repeat_last_n = std::stoi(options.repeat_last_n); - CLI_LOG("Updated repeat_last_n to: " << options.repeat_last_n); - } - if (!options.repeat_penalty.empty()) { - model_config.repeat_penalty = std::stof(options.repeat_penalty); - CLI_LOG("Updated repeat_penalty to: " << options.repeat_penalty); - } - if (!options.mirostat.empty()) { - model_config.mirostat = - (options.mirostat == "true" || options.mirostat == "1"); - CLI_LOG("Updated mirostat to: " << options.mirostat); - } - if (!options.mirostat_tau.empty()) { - model_config.mirostat_tau = std::stof(options.mirostat_tau); - CLI_LOG("Updated mirostat_tau to: " << options.mirostat_tau); - } - if (!options.mirostat_eta.empty()) { - model_config.mirostat_eta = std::stof(options.mirostat_eta); - CLI_LOG("Updated mirostat_eta to: " << options.mirostat_eta); - } - if (!options.penalize_nl.empty()) { - model_config.penalize_nl = - (options.penalize_nl == "true" || options.penalize_nl == "1"); - CLI_LOG("Updated penalize_nl to: " << options.penalize_nl); - } - if (!options.ignore_eos.empty()) { - model_config.ignore_eos = - (options.ignore_eos == "true" || options.ignore_eos == "1"); - CLI_LOG("Updated ignore_eos to: " << options.ignore_eos); - } - if (!options.n_probs.empty()) { - model_config.n_probs = std::stoi(options.n_probs); - CLI_LOG("Updated n_probs to: " << options.n_probs); - } - if (!options.min_keep.empty()) { - model_config.min_keep = std::stoi(options.min_keep); - CLI_LOG("Updated min_keep to: " << options.min_keep); - } - if (!options.grammar.empty()) { - model_config.grammar = options.grammar; - CLI_LOG("Updated grammar to: " << options.grammar); - } - yaml_handler.UpdateModelConfig(model_config); - yaml_handler.WriteYamlFile(model_entry.path_to_model_yaml); - CLI_LOG("Successfully update model ID '" + model_handle_ + "'!"); + yaml_handler_.UpdateModelConfig(model_config_); + yaml_handler_.WriteYamlFile(model_entry.path_to_model_yaml); + CLI_LOG("Successfully updated model ID '" + model_handle_ + "'!"); } catch (const std::exception& e) { - CLI_LOG("Fail to update model with model ID '" + model_handle_ + + CLI_LOG("Failed to update model with model ID '" + model_handle_ + "': " + e.what()); } } +void ModelUpdCmd::UpdateConfig(const std::string& key, + const std::string& value) { + static const std::unordered_map< + std::string, + std::function> + updaters = { + {"name", + [](ModelUpdCmd* self, const std::string&, const std::string& v) { + self->model_config_.name = v; + }}, + {"model", + [](ModelUpdCmd* self, const std::string&, const std::string& v) { + self->model_config_.model = v; + }}, + {"version", + [](ModelUpdCmd* self, const std::string&, const std::string& v) { + self->model_config_.version = v; + }}, + {"stop", &ModelUpdCmd::UpdateVectorField}, + {"top_p", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateNumericField( + k, v, [self](float f) { self->model_config_.top_p = f; }); + }}, + {"temperature", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateNumericField(k, v, [self](float f) { + self->model_config_.temperature = f; + }); + }}, + {"frequency_penalty", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateNumericField(k, v, [self](float f) { + self->model_config_.frequency_penalty = f; + }); + }}, + {"presence_penalty", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateNumericField(k, v, [self](float f) { + self->model_config_.presence_penalty = f; + }); + }}, + {"max_tokens", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateNumericField(k, v, [self](float f) { + self->model_config_.max_tokens = static_cast(f); + }); + }}, + {"stream", + [](ModelUpdCmd* self, const std::string& k, const std::string& v) { + self->UpdateBooleanField( + k, v, [self](bool b) { self->model_config_.stream = b; }); + }}, + // Add more fields here... + }; + + if (auto it = updaters.find(key); it != updaters.end()) { + it->second(this, key, value); + LogUpdate(key, value); + } +} + +void ModelUpdCmd::UpdateVectorField(const std::string& key, + const std::string& value) { + std::vector tokens; + std::istringstream iss(value); + std::string token; + while (std::getline(iss, token, ',')) { + tokens.push_back(token); + } + model_config_.stop = tokens; +} + +void ModelUpdCmd::UpdateNumericField(const std::string& key, + const std::string& value, + std::function setter) { + try { + float numericValue = std::stof(value); + setter(numericValue); + } catch (const std::exception& e) { + CLI_LOG("Failed to parse numeric value for " << key << ": " << e.what()); + } +} + +void ModelUpdCmd::UpdateBooleanField(const std::string& key, + const std::string& value, + std::function setter) { + bool boolValue = (value == "true" || value == "1"); + setter(boolValue); +} + +void ModelUpdCmd::LogUpdate(const std::string& key, const std::string& value) { + CLI_LOG("Updated " << key << " to: " << value); +} + } // namespace commands \ No newline at end of file diff --git a/engine/commands/model_upd_cmd.h b/engine/commands/model_upd_cmd.h index 0e94767aa..51f5a88d3 100644 --- a/engine/commands/model_upd_cmd.h +++ b/engine/commands/model_upd_cmd.h @@ -5,60 +5,26 @@ #include #include #include "config/model_config.h" +#include "utils/modellist_utils.h" +#include "config/yaml_config.h" namespace commands { -struct ModelUpdateOptions { - std::string name; - std::string model; - std::string version; - std::string stop; - std::string top_p; - std::string temperature; - std::string frequency_penalty; - std::string presence_penalty; - std::string max_tokens; - std::string stream; - std::string ngl; - std::string ctx_len; - std::string engine; - std::string prompt_template; - std::string system_template; - std::string user_template; - std::string ai_template; - std::string os; - std::string gpu_arch; - std::string quantization_method; - std::string precision; - std::string tp; - std::string trtllm_version; - std::string text_model; - std::string files; - std::string created; - std::string object; - std::string owned_by; - std::string seed; - std::string dynatemp_range; - std::string dynatemp_exponent; - std::string top_k; - std::string min_p; - std::string tfs_z; - std::string typ_p; - std::string repeat_last_n; - std::string repeat_penalty; - std::string mirostat; - std::string mirostat_tau; - std::string mirostat_eta; - std::string penalize_nl; - std::string ignore_eos; - std::string n_probs; - std::string min_keep; - std::string grammar; -}; class ModelUpdCmd { public: ModelUpdCmd(std::string model_handle); - void Exec(const ModelUpdateOptions& options); + void Exec(const std::unordered_map& options); private: std::string model_handle_; + config::ModelConfig model_config_; + config::YamlHandler yaml_handler_; + modellist_utils::ModelListUtils model_list_utils_; + + void UpdateConfig(const std::string& key, const std::string& value); + void UpdateVectorField(const std::string& key, const std::string& value); + void UpdateNumericField(const std::string& key, const std::string& value, + std::function setter); + void UpdateBooleanField(const std::string& key, const std::string& value, + std::function setter); + void LogUpdate(const std::string& key, const std::string& value); }; } // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 9d4aa5385..c3f60e2f3 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -471,95 +471,27 @@ void CommandLineParser::ModelUpdate(CLI::App* parent) { ->required(); // Add options dynamically - model_update_cmd->add_option("--name", model_update_options_.name, "Name"); - model_update_cmd->add_option("--model", model_update_options_.model, "Model"); - model_update_cmd->add_option("--version", model_update_options_.version, - "Version"); - model_update_cmd->add_option("--stop", model_update_options_.stop, "Stop"); - model_update_cmd->add_option("--top_p", model_update_options_.top_p, "Top P"); - model_update_cmd->add_option( - "--temperature", model_update_options_.temperature, "Temperature"); - model_update_cmd->add_option("--frequency_penalty", - model_update_options_.frequency_penalty, - "Frequency Penalty"); - model_update_cmd->add_option("--presence_penalty", - model_update_options_.presence_penalty, - "Presence Penalty"); - model_update_cmd->add_option("--max_tokens", model_update_options_.max_tokens, - "Max Tokens"); - model_update_cmd->add_option("--stream", model_update_options_.stream, - "Stream"); - model_update_cmd->add_option("--ngl", model_update_options_.ngl, "NGL"); - model_update_cmd->add_option("--ctx_len", model_update_options_.ctx_len, - "Context Length"); - model_update_cmd->add_option("--engine", model_update_options_.engine, - "Engine"); - model_update_cmd->add_option("--prompt_template", - model_update_options_.prompt_template, - "Prompt Template"); - model_update_cmd->add_option("--system_template", - model_update_options_.system_template, - "System Template"); - model_update_cmd->add_option( - "--user_template", model_update_options_.user_template, "User Template"); - model_update_cmd->add_option( - "--ai_template", model_update_options_.ai_template, "AI Template"); - model_update_cmd->add_option("--os", model_update_options_.os, "OS"); - model_update_cmd->add_option("--gpu_arch", model_update_options_.gpu_arch, - "GPU Architecture"); - model_update_cmd->add_option("--quantization_method", - model_update_options_.quantization_method, - "Quantization Method"); - model_update_cmd->add_option("--precision", model_update_options_.precision, - "Precision"); - model_update_cmd->add_option("--tp", model_update_options_.tp, "TP"); - model_update_cmd->add_option("--trtllm_version", - model_update_options_.trtllm_version, - "TRTLLM Version"); - model_update_cmd->add_option("--text_model", model_update_options_.text_model, - "Text Model"); - model_update_cmd->add_option("--files", model_update_options_.files, "Files"); - model_update_cmd->add_option("--created", model_update_options_.created, - "Created"); - model_update_cmd->add_option("--object", model_update_options_.object, - "Object"); - model_update_cmd->add_option("--owned_by", model_update_options_.owned_by, - "Owned By"); - model_update_cmd->add_option("--seed", model_update_options_.seed, "Seed"); - model_update_cmd->add_option("--dynatemp_range", - model_update_options_.dynatemp_range, - "Dynatemp Range"); - model_update_cmd->add_option("--dynatemp_exponent", - model_update_options_.dynatemp_exponent, - "Dynatemp Exponent"); - model_update_cmd->add_option("--top_k", model_update_options_.top_k, "Top K"); - model_update_cmd->add_option("--min_p", model_update_options_.min_p, "Min P"); - model_update_cmd->add_option("--tfs_z", model_update_options_.tfs_z, "TFS Z"); - model_update_cmd->add_option("--typ_p", model_update_options_.typ_p, "Typ P"); - model_update_cmd->add_option( - "--repeat_last_n", model_update_options_.repeat_last_n, "Repeat Last N"); - model_update_cmd->add_option("--repeat_penalty", - model_update_options_.repeat_penalty, - "Repeat Penalty"); - model_update_cmd->add_option("--mirostat", model_update_options_.mirostat, - "Mirostat"); - model_update_cmd->add_option( - "--mirostat_tau", model_update_options_.mirostat_tau, "Mirostat Tau"); - model_update_cmd->add_option( - "--mirostat_eta", model_update_options_.mirostat_eta, "Mirostat Eta"); - model_update_cmd->add_option( - "--penalize_nl", model_update_options_.penalize_nl, "Penalize NL"); - model_update_cmd->add_option("--ignore_eos", model_update_options_.ignore_eos, - "Ignore EOS"); - model_update_cmd->add_option("--n_probs", model_update_options_.n_probs, - "N Probs"); - model_update_cmd->add_option("--min_keep", model_update_options_.min_keep, - "Min Keep"); - model_update_cmd->add_option("--grammar", model_update_options_.grammar, - "Grammar"); - - model_update_cmd->callback([this]() { + + +// Add options dynamically +std::vector option_names = { + "name", "model", "version", "stop", "top_p", "temperature", + "frequency_penalty", "presence_penalty", "max_tokens", "stream", + "ngl", "ctx_len", "engine", "prompt_template", "system_template", + "user_template", "ai_template", "os", "gpu_arch", "quantization_method", + "precision", "tp", "trtllm_version", "text_model", "files", "created", + "object", "owned_by", "seed", "dynatemp_range", "dynatemp_exponent", + "top_k", "min_p", "tfs_z", "typ_p", "repeat_last_n", "repeat_penalty", + "mirostat", "mirostat_tau", "mirostat_eta", "penalize_nl", "ignore_eos", + "n_probs", "min_keep", "grammar" +}; + +for (const auto& option_name : option_names) { + model_update_cmd->add_option("--" + option_name, cml_data_.model_update_options[option_name], option_name); +} + +model_update_cmd->callback([this]() { commands::ModelUpdCmd command(cml_data_.model_id); - command.Exec(model_update_options_); - }); + command.Exec(cml_data_.model_update_options); +}); } \ No newline at end of file diff --git a/engine/controllers/command_line_parser.h b/engine/controllers/command_line_parser.h index 675c74cad..aaa24e064 100644 --- a/engine/controllers/command_line_parser.h +++ b/engine/controllers/command_line_parser.h @@ -41,7 +41,7 @@ class CommandLineParser { bool check_upd = true; int port; config_yaml_utils::CortexConfig config; + std::unordered_map model_update_options; }; CmlData cml_data_; - commands::ModelUpdateOptions model_update_options_; }; From 5a4eb0c6f92d762c15d8b02d2499e3720e4def34 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Tue, 24 Sep 2024 11:24:55 +0700 Subject: [PATCH 6/9] Format code --- engine/controllers/command_line_parser.cc | 73 ++++++++++++++++------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index c3f60e2f3..1422ced17 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -471,27 +471,60 @@ void CommandLineParser::ModelUpdate(CLI::App* parent) { ->required(); // Add options dynamically + std::vector option_names = {"name", + "model", + "version", + "stop", + "top_p", + "temperature", + "frequency_penalty", + "presence_penalty", + "max_tokens", + "stream", + "ngl", + "ctx_len", + "engine", + "prompt_template", + "system_template", + "user_template", + "ai_template", + "os", + "gpu_arch", + "quantization_method", + "precision", + "tp", + "trtllm_version", + "text_model", + "files", + "created", + "object", + "owned_by", + "seed", + "dynatemp_range", + "dynatemp_exponent", + "top_k", + "min_p", + "tfs_z", + "typ_p", + "repeat_last_n", + "repeat_penalty", + "mirostat", + "mirostat_tau", + "mirostat_eta", + "penalize_nl", + "ignore_eos", + "n_probs", + "min_keep", + "grammar"}; + + for (const auto& option_name : option_names) { + model_update_cmd->add_option("--" + option_name, + cml_data_.model_update_options[option_name], + option_name); + } - -// Add options dynamically -std::vector option_names = { - "name", "model", "version", "stop", "top_p", "temperature", - "frequency_penalty", "presence_penalty", "max_tokens", "stream", - "ngl", "ctx_len", "engine", "prompt_template", "system_template", - "user_template", "ai_template", "os", "gpu_arch", "quantization_method", - "precision", "tp", "trtllm_version", "text_model", "files", "created", - "object", "owned_by", "seed", "dynatemp_range", "dynatemp_exponent", - "top_k", "min_p", "tfs_z", "typ_p", "repeat_last_n", "repeat_penalty", - "mirostat", "mirostat_tau", "mirostat_eta", "penalize_nl", "ignore_eos", - "n_probs", "min_keep", "grammar" -}; - -for (const auto& option_name : option_names) { - model_update_cmd->add_option("--" + option_name, cml_data_.model_update_options[option_name], option_name); -} - -model_update_cmd->callback([this]() { + model_update_cmd->callback([this]() { commands::ModelUpdCmd command(cml_data_.model_id); command.Exec(cml_data_.model_update_options); -}); + }); } \ No newline at end of file From 6a4e7dd2fc543a2a6d3348ad85e6afac3cce2fd0 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Tue, 24 Sep 2024 11:51:32 +0700 Subject: [PATCH 7/9] Add clean up when finish test --- engine/test/components/test_modellist_utils.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/test/components/test_modellist_utils.cc b/engine/test/components/test_modellist_utils.cc index 68b06483d..40fb20e54 100644 --- a/engine/test/components/test_modellist_utils.cc +++ b/engine/test/components/test_modellist_utils.cc @@ -128,4 +128,6 @@ TEST_F(ModelListUtilsTestSuite, TestHasModel) { EXPECT_TRUE(model_list_.HasModel("test_model_id")); EXPECT_TRUE(model_list_.HasModel("test_alias")); EXPECT_FALSE(model_list_.HasModel("non_existent_model")); + // Clean up + model_list_.DeleteModelEntry("test_model_id"); } \ No newline at end of file From fe1c7004b49175da11e07de27a64f82a3db6cc4c Mon Sep 17 00:00:00 2001 From: Thuandz Date: Tue, 24 Sep 2024 11:57:49 +0700 Subject: [PATCH 8/9] remove model.list after finish test --- engine/test/components/test_modellist_utils.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/test/components/test_modellist_utils.cc b/engine/test/components/test_modellist_utils.cc index 40fb20e54..ea4827493 100644 --- a/engine/test/components/test_modellist_utils.cc +++ b/engine/test/components/test_modellist_utils.cc @@ -19,6 +19,7 @@ class ModelListUtilsTestSuite : public ::testing::Test { void TearDown() { // Clean up the temporary directory + std::remove((file_manager_utils::GetModelsContainerPath() / "model.list").c_str()); } TEST_F(ModelListUtilsTestSuite, TestAddModelEntry) { EXPECT_TRUE(model_list_.AddModelEntry(kTestModel)); From a6b2bc4c5c4aac026c5db37ef2900c4e1fcb8093 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Tue, 24 Sep 2024 12:02:26 +0700 Subject: [PATCH 9/9] Fix windows CI build --- engine/test/components/test_modellist_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/test/components/test_modellist_utils.cc b/engine/test/components/test_modellist_utils.cc index ea4827493..d1dbf91e3 100644 --- a/engine/test/components/test_modellist_utils.cc +++ b/engine/test/components/test_modellist_utils.cc @@ -19,7 +19,7 @@ class ModelListUtilsTestSuite : public ::testing::Test { void TearDown() { // Clean up the temporary directory - std::remove((file_manager_utils::GetModelsContainerPath() / "model.list").c_str()); + std::remove((file_manager_utils::GetModelsContainerPath() / "model.list").string().c_str()); } TEST_F(ModelListUtilsTestSuite, TestAddModelEntry) { EXPECT_TRUE(model_list_.AddModelEntry(kTestModel));