From abb5d7f43a202e75bf449fbe0489f28238f5c2fb Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 26 Sep 2024 00:46:56 +0700 Subject: [PATCH 1/6] fix/mistral-nemo-chat-template --- engine/config/chat_template_renderer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/config/chat_template_renderer.h b/engine/config/chat_template_renderer.h index f40894f7b..63a47ecf3 100644 --- a/engine/config/chat_template_renderer.h +++ b/engine/config/chat_template_renderer.h @@ -123,7 +123,7 @@ static int32_t llama_chat_apply_template_internal( ss << content << "\n"; } } else if (role == "user") { - ss << content << " [/INST]"; + ss << "[INST] " << content << " [/INST]"; } else { ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << ""; From dc45f0f08f50263008a1cf627eedb02a6a097778 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 30 Sep 2024 11:54:26 +0700 Subject: [PATCH 2/6] feat/config-engines-log-path --- engine/controllers/server.cc | 2 +- engine/test/components/test_cortex_config.cc | 38 +++++++++++++++----- engine/utils/config_yaml_utils.h | 18 +++++++++- engine/utils/cortex_utils.h | 3 ++ engine/utils/file_manager_utils.h | 4 +++ 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/engine/controllers/server.cc b/engine/controllers/server.cc index 2e8a93b8c..2cf6800df 100644 --- a/engine/controllers/server.cc +++ b/engine/controllers/server.cc @@ -348,7 +348,7 @@ void server::LoadModel(const HttpRequestPtr& req, auto config = file_manager_utils::GetCortexConfig(); if (en->IsSupported("SetFileLogger")) { en->SetFileLogger(config.maxLogLines, config.logFolderPath + "/" + - cortex_utils::logs_base_name); + config.logLlamaCppPath); } else { LOG_WARN << "Method SetFileLogger is not supported yet"; } diff --git a/engine/test/components/test_cortex_config.cc b/engine/test/components/test_cortex_config.cc index 513716940..865ebeb59 100644 --- a/engine/test/components/test_cortex_config.cc +++ b/engine/test/components/test_cortex_config.cc @@ -9,10 +9,16 @@ class CortexConfigTest : public ::testing::Test { void SetUp() override { // Set up default configuration - default_config = { - "default_log_path", "default_data_path", 1000, - kDefaultHost, kDefaultPort, kDefaultCheckedForUpdateAt, - kDefaultLatestRelease}; + default_config = {"default_log_path", + "default_llamacpp_log_path", + "default_tensorrtllm_log_path", + "default_onnx_log_path", + "default_data_path", + 1000, + kDefaultHost, + kDefaultPort, + kDefaultCheckedForUpdateAt, + kDefaultLatestRelease}; } void TearDown() override { @@ -24,8 +30,16 @@ class CortexConfigTest : public ::testing::Test { }; TEST_F(CortexConfigTest, DumpYamlConfig_WritesCorrectly) { - CortexConfig config = {"log_path", "data_path", 5000, "localhost", - "8080", 123456789, "v1.0.0"}; + CortexConfig config = {"log_path", + "default_llamacpp_log_path", + "default_tensorrtllm_log_path", + "default_onnx_log_path", + "data_path", + 5000, + "localhost", + "8080", + 123456789, + "v1.0.0"}; DumpYamlConfig(config, test_file_path); @@ -43,8 +57,16 @@ TEST_F(CortexConfigTest, DumpYamlConfig_WritesCorrectly) { TEST_F(CortexConfigTest, FromYaml_ReadsCorrectly) { // First, create a valid YAML configuration file - CortexConfig config = {"log_path", "data_path", 5000, "localhost", - "8080", 123456789, "v1.0.0"}; + CortexConfig config = {"log_path", + "default_llamacpp_log_path", + "default_tensorrtllm_log_path", + "default_onnx_log_path", + "data_path", + 5000, + "localhost", + "8080", + 123456789, + "v1.0.0"}; DumpYamlConfig(config, test_file_path); diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 22672b5ac..8ceabe2eb 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -9,6 +9,9 @@ namespace config_yaml_utils { struct CortexConfig { std::string logFolderPath; + std::string logLlamaCppPath; + std::string logTensorrtLLMPath; + std::string logOnnxPath; std::string dataFolderPath; int maxLogLines; std::string apiServerHost; @@ -35,6 +38,9 @@ inline void DumpYamlConfig(const CortexConfig& config, } YAML::Node node; node["logFolderPath"] = config.logFolderPath; + node["logLlamaCppPath"] = config.logLlamaCppPath; + node["logTensorrtLLMPath"] = config.logTensorrtLLMPath; + node["logOnnxPath"] = config.logOnnxPath; node["dataFolderPath"] = config.dataFolderPath; node["maxLogLines"] = config.maxLogLines; node["apiServerHost"] = config.apiServerHost; @@ -63,12 +69,22 @@ inline CortexConfig FromYaml(const std::string& path, (!node["logFolderPath"] || !node["dataFolderPath"] || !node["maxLogLines"] || !node["apiServerHost"] || !node["apiServerPort"] || !node["checkedForUpdateAt"] || - !node["latestRelease"]); + !node["latestRelease"] || !node["logLlamaCppPath"] || + !node["logOnnxPath"] || !node["logTensorrtLLMPath"]); CortexConfig config = { .logFolderPath = node["logFolderPath"] ? node["logFolderPath"].as() : default_cfg.logFolderPath, + .logLlamaCppPath = node["logLlamaCppPath"] + ? node["logLlamaCppPath"].as() + : default_cfg.logLlamaCppPath, + .logTensorrtLLMPath = node["logTensorrtLLMPath"] + ? node["logTensorrtLLMPath"].as() + : default_cfg.logTensorrtLLMPath, + .logOnnxPath = node["logOnnxPath"] + ? node["logOnnxPath"].as() + : default_cfg.logOnnxPath, .dataFolderPath = node["dataFolderPath"] ? node["dataFolderPath"].as() : default_cfg.dataFolderPath, diff --git a/engine/utils/cortex_utils.h b/engine/utils/cortex_utils.h index 9673f0c1a..d81a4d9e9 100644 --- a/engine/utils/cortex_utils.h +++ b/engine/utils/cortex_utils.h @@ -35,6 +35,9 @@ constexpr static auto kTensorrtLlmPath = "/engines/cortex.tensorrt-llm"; inline std::string models_folder = "./models"; inline std::string logs_folder = "./logs"; inline std::string logs_base_name = "./logs/cortex.log"; +inline std::string logs_llamacpp_base_name = "./logs/cortex.log"; +inline std::string logs_tensorrtllm_base_name = "./logs/cortex.log"; +inline std::string logs_onnx_base_name = "./logs/cortex.log"; inline std::string logs_cli_base_name = "./logs/cortex-cli.log"; inline std::string extractBase64(const std::string& input) { diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index 3dccb042a..a07b9cb21 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -5,6 +5,7 @@ #include "logging_utils.h" #include "services/download_service.h" #include "utils/config_yaml_utils.h" +#include "utils/cortex_utils.h" #if defined(__APPLE__) && defined(__MACH__) #include @@ -156,6 +157,9 @@ inline config_yaml_utils::CortexConfig GetCortexConfig() { file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name; auto default_cfg = config_yaml_utils::CortexConfig{ .logFolderPath = default_data_folder_path.string(), + .logLlamaCppPath = cortex_utils::logs_llamacpp_base_name, + .logTensorrtLLMPath = cortex_utils::logs_tensorrtllm_base_name, + .logOnnxPath = cortex_utils::logs_onnx_base_name, .dataFolderPath = default_data_folder_path.string(), .maxLogLines = config_yaml_utils::kDefaultMaxLines, .apiServerHost = config_yaml_utils::kDefaultHost, From fea8dbbbf0a4914520a8ac9b9c73bcdcb19bdc4f Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 30 Sep 2024 12:20:55 +0700 Subject: [PATCH 3/6] Fix: use file system path instead of string concat --- engine/controllers/server.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/controllers/server.cc b/engine/controllers/server.cc index 2cf6800df..f19b1412e 100644 --- a/engine/controllers/server.cc +++ b/engine/controllers/server.cc @@ -347,8 +347,8 @@ void server::LoadModel(const HttpRequestPtr& req, if (engine_type == kLlamaEngine) { //fix for llamacpp engine first auto config = file_manager_utils::GetCortexConfig(); if (en->IsSupported("SetFileLogger")) { - en->SetFileLogger(config.maxLogLines, config.logFolderPath + "/" + - config.logLlamaCppPath); + en->SetFileLogger(config.maxLogLines, (std::filesystem::path(config.logFolderPath) / + std::filesystem::path(config.logLlamaCppPath)).string()); } else { LOG_WARN << "Method SetFileLogger is not supported yet"; } From ab3a51a8877569445fc82d5f5b3dc95c4c12ebdf Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 30 Sep 2024 12:46:27 +0700 Subject: [PATCH 4/6] Fix: CI build window --- engine/utils/file_manager_utils.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index a07b9cb21..47d08df1f 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -1,11 +1,14 @@ #pragma once +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on #include #include #include #include "logging_utils.h" #include "services/download_service.h" #include "utils/config_yaml_utils.h" -#include "utils/cortex_utils.h" + #if defined(__APPLE__) && defined(__MACH__) #include From 287a39152dc09cc321581b0850f691d683238e69 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 30 Sep 2024 12:50:05 +0700 Subject: [PATCH 5/6] Fix: CI build window --- engine/utils/file_manager_utils.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index 47d08df1f..2db1ad5fe 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -1,14 +1,13 @@ #pragma once -// clang-format off -#include "utils/cortex_utils.h" -// clang-format on #include #include #include #include "logging_utils.h" #include "services/download_service.h" #include "utils/config_yaml_utils.h" - +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on #if defined(__APPLE__) && defined(__MACH__) #include From a230c79dd79d4abe8a14f9d1c47fd1b46b925f3a Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 30 Sep 2024 13:08:25 +0700 Subject: [PATCH 6/6] Fix: CI build windows --- engine/utils/cortex_utils.h | 3 --- engine/utils/file_manager_utils.h | 13 +++++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/engine/utils/cortex_utils.h b/engine/utils/cortex_utils.h index d81a4d9e9..9673f0c1a 100644 --- a/engine/utils/cortex_utils.h +++ b/engine/utils/cortex_utils.h @@ -35,9 +35,6 @@ constexpr static auto kTensorrtLlmPath = "/engines/cortex.tensorrt-llm"; inline std::string models_folder = "./models"; inline std::string logs_folder = "./logs"; inline std::string logs_base_name = "./logs/cortex.log"; -inline std::string logs_llamacpp_base_name = "./logs/cortex.log"; -inline std::string logs_tensorrtllm_base_name = "./logs/cortex.log"; -inline std::string logs_onnx_base_name = "./logs/cortex.log"; inline std::string logs_cli_base_name = "./logs/cortex-cli.log"; inline std::string extractBase64(const std::string& input) { diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index 2db1ad5fe..6e8dacced 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -5,9 +5,7 @@ #include "logging_utils.h" #include "services/download_service.h" #include "utils/config_yaml_utils.h" -// clang-format off -#include "utils/cortex_utils.h" -// clang-format on + #if defined(__APPLE__) && defined(__MACH__) #include @@ -23,6 +21,9 @@ constexpr std::string_view kDefaultConfigurationPath = "user_home"; constexpr std::string_view kProdVariant = "prod"; constexpr std::string_view kBetaVariant = "beta"; constexpr std::string_view kNightlyVariant = "nightly"; +constexpr char kLogsLlamacppBaseName[] = "./logs/cortex.log"; +constexpr char kLogsTensorrtllmBaseName[] = "./logs/cortex.log"; +constexpr char kLogsOnnxBaseName[] = "./logs/cortex.log"; inline std::filesystem::path GetExecutableFolderContainerPath() { #if defined(__APPLE__) && defined(__MACH__) @@ -159,9 +160,9 @@ inline config_yaml_utils::CortexConfig GetCortexConfig() { file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name; auto default_cfg = config_yaml_utils::CortexConfig{ .logFolderPath = default_data_folder_path.string(), - .logLlamaCppPath = cortex_utils::logs_llamacpp_base_name, - .logTensorrtLLMPath = cortex_utils::logs_tensorrtllm_base_name, - .logOnnxPath = cortex_utils::logs_onnx_base_name, + .logLlamaCppPath = kLogsLlamacppBaseName, + .logTensorrtLLMPath = kLogsTensorrtllmBaseName, + .logOnnxPath = kLogsOnnxBaseName, .dataFolderPath = default_data_folder_path.string(), .maxLogLines = config_yaml_utils::kDefaultMaxLines, .apiServerHost = config_yaml_utils::kDefaultHost,