Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit e209137

Browse files
namchuainamchuai
committed
update
1 parent ed0cb6d commit e209137

File tree

10 files changed

+124
-34
lines changed

10 files changed

+124
-34
lines changed

‎engine/commands/engine_get_cmd.cc‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,64 @@
11
#include "engine_get_cmd.h"
2+
#include <iostream>
3+
#include <tabulate/table.hpp>
4+
#include "utils/file_manager_utils.h"
5+
#include "utils/logging_utils.h"
26

37
namespace commands {
48

5-
void EngineGetCmd::Exec() const {}
9+
void EngineGetCmd::Exec() const {
10+
CTL_INF("[EneingeGetCmd] engine: " << engine_);
11+
12+
auto ecp = file_manager_utils::GetEnginesContainerPath();
13+
std::string onnx_status{"not_supported"};
14+
std::string llamacpp_status = std::filesystem::exists(ecp / "cortex.llamacpp")
15+
? "ready"
16+
: "not_initialized";
17+
std::string tensorrt_status{"not_supported"};
18+
19+
#ifdef _WIN32
20+
onnx_status = std::filesystem::exists(ecp / "cortex.onnx")
21+
? "ready"
22+
: "not_initialized";
23+
tensorrt_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
24+
? "ready"
25+
: "not_initialized";
26+
#elif defined(__linux__)
27+
tensorrt_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
28+
? "ready"
29+
: "not_initialized";
30+
#endif
31+
std::vector<EngineInfo> engines = {
32+
{.name = "cortex.onnx",
33+
.description = "This extension enables chat completion API calls using "
34+
"the Onnx engine",
35+
.version = "0.0.1",
36+
.product_name = "Onnx Inference Engine",
37+
.status = onnx_status},
38+
{.name = "cortex.llamacpp",
39+
.description = "This extension enables chat completion API calls using "
40+
"the LlamaCPP engine",
41+
.version = "0.0.1",
42+
.product_name = "LlamaCPP Inference Engine",
43+
.status = llamacpp_status},
44+
{.name = "cortex.tensorrt-llm",
45+
.description = "This extension enables chat completion API calls using "
46+
"the TensorrtLLM engine",
47+
.version = "0.0.1",
48+
.product_name = "TensorrtLLM Inference Engine",
49+
.status = tensorrt_status},
50+
};
51+
52+
tabulate::Table table;
53+
table.add_row({"name", "description", "version", "product name", "status"});
54+
table.format().font_color(tabulate::Color::green);
55+
for (auto& engine : engines) {
56+
if (engine.name == engine_) {
57+
table.add_row({engine.name, engine.description, engine.version,
58+
engine.product_name, engine.status});
59+
}
60+
}
61+
62+
std::cout << table << std::endl;
63+
}
664
}; // namespace commands

‎engine/commands/engine_get_cmd.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,23 @@
11
#pragma once
2+
#include <string>
23

34
namespace commands {
45
class EngineGetCmd {
6+
struct EngineInfo {
7+
std::string name;
8+
std::string description;
9+
std::string version;
10+
std::string product_name;
11+
std::string status;
12+
};
13+
514
public:
15+
EngineGetCmd(const std::string& engine) : engine_{engine} {};
16+
617
void Exec() const;
18+
19+
private:
20+
std::string engine_;
721
};
822

923
} // namespace commands

‎engine/commands/engine_list_cmd.cc‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// clang-format off
2-
#include "utils/cortex_utils.h"
3-
// clang-format on
41
#include "engine_list_cmd.h"
52
#include <filesystem>
63
#include <tabulate/table.hpp>
@@ -14,16 +11,16 @@ bool EngineListCmd::Exec() {
1411
std::string llamacpp_status = std::filesystem::exists(ecp / "cortex.llamacpp")
1512
? "ready"
1613
: "not_initialized";
17-
std::string tenssorrt_status{"not_supported"};
14+
std::string tensorrt_status{"not_supported"};
1815
#ifdef _WIN32
1916
onnx_status = std::filesystem::exists(ecp / "cortex.onnx")
2017
? "ready"
2118
: "not_initialized";
22-
tenssort_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
19+
tensorrt_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
2320
? "ready"
2421
: "not_initialized";
25-
#elif __linux__
26-
tenssort_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
22+
#elif defined(__linux__)
23+
tensorrt_status = std::filesystem::exists(ecp / "cortex.tensorrt-llm")
2724
? "ready"
2825
: "not_initialized";
2926
#endif
@@ -46,7 +43,7 @@ bool EngineListCmd::Exec() {
4643
table.add_row({"3", "cortex.tensorrt-llm",
4744
"This extension enables chat completion API calls using the "
4845
"TensorrtLLM engine",
49-
"0.0.1", "TensorrtLLM Inference Engine", tenssorrt_status});
46+
"0.0.1", "TensorrtLLM Inference Engine", tensorrt_status});
5047

5148
for (int i = 0; i < 6; i++) {
5249
table[0][i]

‎engine/controllers/command_line_parser.cc‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "command_line_parser.h"
22
#include "commands/chat_cmd.h"
33
#include "commands/cmd_info.h"
4+
#include "commands/engine_get_cmd.h"
45
#include "commands/engine_init_cmd.h"
56
#include "commands/engine_list_cmd.h"
67
#include "commands/engine_uninstall_cmd.h"
@@ -125,11 +126,11 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
125126
command.Exec();
126127
});
127128

128-
auto get_engine_cmd = engines_cmd->add_subcommand("get", "Get an engine");
129+
EngineManagement(engines_cmd, "cortex.llamacpp", version);
130+
EngineManagement(engines_cmd, "cortex.onnx", version);
131+
EngineManagement(engines_cmd, "cortex.tensorrt-llm", version);
129132

130-
EngineInstall(engines_cmd, "cortex.llamacpp", version);
131-
EngineInstall(engines_cmd, "cortex.onnx", version);
132-
EngineInstall(engines_cmd, "cortex.tensorrt-llm", version);
133+
EngineGet(engines_cmd);
133134
}
134135

135136
{
@@ -158,9 +159,9 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
158159
return true;
159160
}
160161

161-
void CommandLineParser::EngineInstall(CLI::App* parent,
162-
const std::string& engine_name,
163-
std::string& version) {
162+
void CommandLineParser::EngineManagement(CLI::App* parent,
163+
const std::string& engine_name,
164+
std::string& version) {
164165
auto engine_cmd =
165166
parent->add_subcommand(engine_name, "Manage " + engine_name + " engine");
166167

@@ -181,3 +182,17 @@ void CommandLineParser::EngineInstall(CLI::App* parent,
181182
cmd.Exec();
182183
});
183184
}
185+
186+
void CommandLineParser::EngineGet(CLI::App* parent) {
187+
auto get_cmd = parent->add_subcommand("get", "Get an engine info");
188+
189+
for (auto& engine : supportedEngines_) {
190+
std::string engine_name{engine};
191+
std::string desc = "Get " + engine_name + " status";
192+
auto engine_get_cmd = get_cmd->add_subcommand(engine, desc);
193+
engine_get_cmd->callback([engine_name] {
194+
commands::EngineGetCmd cmd(engine_name);
195+
cmd.Exec();
196+
});
197+
}
198+
}

‎engine/controllers/command_line_parser.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <array>
34
#include "CLI/CLI.hpp"
45

56
class CommandLineParser {
@@ -8,8 +9,14 @@ class CommandLineParser {
89
bool SetupCommand(int argc, char** argv);
910

1011
private:
11-
void EngineInstall(CLI::App* parent, const std::string& engine_name,
12-
std::string& version);
12+
void EngineManagement(CLI::App* parent, const std::string& engine_name,
13+
std::string& version);
14+
15+
void EngineGet(CLI::App* parent);
1316

1417
CLI::App app_;
18+
19+
// TODO: move this one to somewhere else
20+
static constexpr std::array<const char*, 3> supportedEngines_ = {
21+
"cortex.llamacpp", "cortex.onnx", "cortex.tensorrt-llm"};
1522
};

‎engine/controllers/engines.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#include "engines.h"
22
#include "utils/archive_utils.h"
3-
#include "utils/file_manager_utils.h"
3+
#include "utils/cortex_utils.h"
44
#include "utils/system_info_utils.h"
55

66
void Engines::InitEngine(const HttpRequestPtr& req,
@@ -172,4 +172,4 @@ void Engines::ListEngine(
172172
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
173173
resp->setStatusCode(k200OK);
174174
callback(resp);
175-
}
175+
}

‎engine/controllers/engines.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,7 @@
22

33
#include <drogon/HttpController.h>
44
#include <trantor/utils/Logger.h>
5-
#include "services/download_service.h"
6-
#include "utils/cortex_utils.h"
75
#include "utils/cortexso_parser.h"
8-
#include "utils/http_util.h"
96

107
using namespace drogon;
118

‎engine/utils/command_executor.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#include <array>
22
#include <cstdio>
3-
#include <iostream>
43
#include <memory>
54
#include <stdexcept>
65
#include <string>
@@ -46,4 +45,4 @@ class CommandExecutor {
4645

4746
private:
4847
std::unique_ptr<FILE, decltype(&PCLOSE)> m_pipe{nullptr, PCLOSE};
49-
};
48+
};

‎engine/utils/cortex_utils.h‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,28 @@
11
#pragma once
22
#include <drogon/HttpClient.h>
33
#include <drogon/HttpResponse.h>
4+
#include <sys/stat.h>
45
#include <algorithm>
56
#include <fstream>
67
#include <iostream>
78
#include <ostream>
8-
#include <regex>
9-
#include <vector>
109
#include <random>
10+
#include <regex>
1111
#include <string>
12-
#include <sys/stat.h>
12+
#include <vector>
1313

1414
// Include platform-specific headers
1515
#ifdef _WIN32
16+
#include <direct.h>
1617
#include <windows.h>
1718
#include <winsock2.h>
18-
#include <direct.h>
1919
#define mkdir _mkdir
2020
#else
2121
#include <dirent.h>
2222
#include <unistd.h>
2323
#endif
2424

2525
#if __APPLE__
26-
#include <limits.h>
2726
#include <mach-o/dyld.h>
2827
#endif
2928

@@ -36,7 +35,7 @@ constexpr static auto kTensorrtLlmPath = "/engines/cortex.tensorrt-llm";
3635
inline std::string models_folder = "./models";
3736
inline std::string logs_folder = "./logs";
3837
inline std::string logs_base_name = "./logs/cortex";
39-
inline size_t log_file_size_limit = 20000000; // ~20 mb
38+
inline size_t log_file_size_limit = 20000000; // ~20 mb
4039

4140
inline std::string extractBase64(const std::string& input) {
4241
std::regex pattern("base64,(.*)");
@@ -273,7 +272,8 @@ inline drogon::HttpResponsePtr CreateCortexHttpResponse() {
273272
return resp;
274273
}
275274

276-
inline drogon::HttpResponsePtr CreateCortexHttpJsonResponse(const Json::Value& data) {
275+
inline drogon::HttpResponsePtr CreateCortexHttpJsonResponse(
276+
const Json::Value& data) {
277277
auto resp = drogon::HttpResponse::newHttpJsonResponse(data);
278278
#ifdef ALLOW_ALL_CORS
279279
LOG_INFO << "Respond for all cors!";
@@ -342,4 +342,4 @@ inline std::string GetCurrentPath() {
342342
}
343343
#endif
344344

345-
} // namespace cortex_utils
345+
} // namespace cortex_utils

‎engine/utils/system_info_utils.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <trantor/utils/Logger.h>
44
#include <regex>
5+
#include <sstream>
56
#include <vector>
67
#include "utils/command_executor.h"
78
#include "utils/logging_utils.h"
@@ -292,7 +293,9 @@ inline std::vector<GpuInfo> GetGpuInfoListVulkan() {
292293
gpuInfoList.push_back(gpuInfo);
293294
++iter;
294295
}
295-
} catch (const std::exception& e) {}
296+
} catch (const std::exception& e) {
297+
LOG_ERROR << "Error: " << e.what();
298+
}
296299

297300
return gpuInfoList;
298301
}

0 commit comments

Comments
 (0)