Merge pull request #1345 from janhq/j/cortex-ps

feat: cortex ps

Author: namchuai

engine/commands/model_stop_cmd.cc

@@ -1,7 +1,6 @@
 #include "model_stop_cmd.h"
 #include "httplib.h"
 #include "nlohmann/json.hpp"
-#include "trantor/utils/Logger.h"
 #include "utils/logging_utils.h"
 
 namespace commands {
@@ -30,4 +29,4 @@ void ModelStopCmd::Exec() {
   }
 }
 
-};  // namespace commands
+};  // namespace commands

engine/commands/model_stop_cmd.h

@@ -1,11 +1,11 @@
 #pragma once
+
 #include <string>
-#include <optional>
 #include "config/model_config.h"
 
 namespace commands {
 
-class ModelStopCmd{
+class ModelStopCmd {
  public:
   ModelStopCmd(std::string host, int port, const config::ModelConfig& mc);
   void Exec();
@@ -15,4 +15,4 @@ class ModelStopCmd{
   int port_;
   const config::ModelConfig& mc_;
 };
-}  // namespace commands
+}  // namespace commands

engine/commands/ps_cmd.cc

@@ -0,0 +1,63 @@
+#include "ps_cmd.h"
+#include <drogon/drogon.h>
+#include <httplib.h>
+#include <string>
+#include <tabulate/table.hpp>
+#include "nlohmann/json.hpp"
+#include "utils/logging_utils.h"
+#include "utils/string_utils.h"
+
+namespace commands {
+
+void PsCmd::Exec(const std::string& host, int port) {
+  auto host_and_port{host + ":" + std::to_string(port)};
+  httplib::Client cli(host_and_port);
+
+  auto res = cli.Get("/inferences/server/models");
+  if (!res || res->status != httplib::StatusCode::OK_200) {
+    CLI_LOG("No model loaded!");
+    return;
+  }
+
+  auto body = nlohmann::json::parse(res->body);
+  auto data = body["data"];
+  std::vector<ModelLoadedStatus> model_status_list;
+  try {
+    for (const auto& item : data) {
+      ModelLoadedStatus model_status;
+      model_status.engine = item["engine"];
+      model_status.model = item["id"];
+      model_status.ram = item["ram"];
+      model_status.start_time = item["start_time"];
+      model_status.vram = item["vram"];
+      model_status_list.push_back(model_status);
+    }
+  } catch (const std::exception& e) {
+    CLI_LOG("Fail to get list model information: " + std::string(e.what()));
+  }
+
+  PrintModelStatusList(model_status_list);
+}
+
+void PsCmd::PrintModelStatusList(
+    const std::vector<ModelLoadedStatus>& model_status_list) const {
+  if (model_status_list.empty()) {
+    CLI_LOG("No model loaded!");
+    return;
+  }
+
+  tabulate::Table table;
+  table.add_row({"Model", "Engine", "RAM", "VRAM", "Up time"});
+  for (const auto& model_status : model_status_list) {
+    table.add_row({
+        model_status.model,
+        model_status.engine,
+        model_status.ram,
+        model_status.vram,
+        string_utils::FormatTimeElapsed(model_status.start_time),
+    });
+  }
+  std::cout << table << std::endl;
+}
+
+};  // namespace commands

engine/commands/ps_cmd.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace commands {
+
+struct ModelLoadedStatus {
+  std::string engine;
+  std::string model;
+  std::string ram;
+  uint64_t start_time;
+  std::string vram;
+};
+
+class PsCmd {
+ public:
+  explicit PsCmd() = default;
+
+  void Exec(const std::string& host, int port);
+
+ private:
+  void PrintModelStatusList(
+      const std::vector<ModelLoadedStatus>& model_status_list) const;
+};
+
+}  // namespace commands

engine/controllers/command_line_parser.cc

@@ -15,6 +15,7 @@
 #include "commands/model_start_cmd.h"
 #include "commands/model_stop_cmd.h"
 #include "commands/model_upd_cmd.h"
+#include "commands/ps_cmd.h"
 #include "commands/run_cmd.h"
 #include "commands/server_start_cmd.h"
 #include "commands/server_stop_cmd.h"
@@ -73,7 +74,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
 #ifdef CORTEX_CPP_VERSION
   if (cml_data_.check_upd) {
     // TODO(sang) find a better way to handle
-    // This is an extremely ungly way to deal with connection 
+    // This is an extremely ungly way to deal with connection
     // hang when network down
     std::atomic<bool> done = false;
     std::thread t([&]() {
@@ -387,6 +388,11 @@ void CommandLineParser::SetupSystemCommands() {
   auto ps_cmd =
       app_.add_subcommand("ps", "Show running models and their status");
   ps_cmd->group(kSystemGroup);
+  ps_cmd->usage("Usage:\n" + commands::GetCortexBinary() + "ps");
+  ps_cmd->callback([&]() {
+    commands::PsCmd().Exec(cml_data_.config.apiServerHost,
+                           std::stoi(cml_data_.config.apiServerPort));
+  });
 
   auto update_cmd = app_.add_subcommand("update", "Update cortex version");
   update_cmd->group(kSystemGroup);

engine/controllers/command_line_parser.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "CLI/CLI.hpp"
-#include "commands/model_upd_cmd.h"
 #include "services/engine_service.h"
 #include "utils/config_yaml_utils.h"
 class CommandLineParser {

engine/utils/string_utils.h

@@ -1,3 +1,5 @@
+#include <chrono>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -29,4 +31,48 @@ inline std::vector<std::string> SplitBy(const std::string& str,
   } while (pos < str.length() && prev < str.length());
   return tokens;
 }
+
+inline uint64_t getCurrentTimeInMilliseconds() {
+  using namespace std::chrono;
+  return duration_cast<milliseconds>(system_clock::now().time_since_epoch())
+      .count();
+}
+
+inline std::string FormatTimeElapsed(uint64_t pastTimestamp) {
+  uint64_t currentTimestamp = getCurrentTimeInMilliseconds();
+  uint64_t milliseconds = currentTimestamp - pastTimestamp;
+
+  // Constants for time units
+  const uint64_t millisInSecond = 1000;
+  const uint64_t millisInMinute = millisInSecond * 60;
+  const uint64_t millisInHour = millisInMinute * 60;
+  const uint64_t millisInDay = millisInHour * 24;
+
+  uint64_t days = milliseconds / millisInDay;
+  milliseconds %= millisInDay;
+
+  uint64_t hours = milliseconds / millisInHour;
+  milliseconds %= millisInHour;
+
+  uint64_t minutes = milliseconds / millisInMinute;
+  milliseconds %= millisInMinute;
+
+  uint64_t seconds = milliseconds / millisInSecond;
+
+  std::ostringstream oss;
+
+  if (days > 0) {
+    oss << days << " day" << (days > 1 ? "s" : "") << ", ";
+  }
+  if (hours > 0 || days > 0) {
+    oss << hours << " hour" << (hours > 1 ? "s" : "") << ", ";
+  }
+  if (minutes > 0 || hours > 0 || days > 0) {
+    oss << minutes << " minute" << (minutes > 1 ? "s" : "") << ", ";
+  }
+
+  oss << seconds << " second" << (seconds > 1 ? "s" : "");
+
+  return oss.str();
+}
 }  // namespace string_utils