feat: unload engine when init engine (#1003)

Author: marknguyen1302

cortex-js/package.json

@@ -53,24 +53,26 @@
     "@nestjs/sequelize": "^10.0.1",
     "@nestjs/swagger": "^7.3.1",
     "@terascope/fetch-github-release": "^0.8.8",
+    "@types/node-os-utils": "^1.3.4",
     "axios": "^1.6.8",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
-    "cortex-cpp": "0.5.0-36",
+    "cortex-cpp": "0.5.0-40",
     "cpu-instructions": "^0.0.11",
     "decompress": "^4.2.1",
     "hyllama": "^0.2.2",
     "js-yaml": "^4.1.0",
     "nest-commander": "^3.13.0",
+    "node-os-utils": "^1.3.7",
     "ora": "5.4.1",
     "readline": "^1.3.0",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "sequelize": "^6.37.3",
     "sequelize-typescript": "^2.1.6",
     "sqlite3": "^5.1.7",
-    "systeminformation": "^5.22.11",
+    "systeminformation": "^5.23.4",
     "ulid": "^2.3.0",
     "uuid": "^9.0.1",
     "whatwg-url": "^14.0.0",

cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts

@@ -41,12 +41,13 @@ export class EnginesInitCommand extends BaseCommand {
     const configs = await fileManagerService.getConfig();
     const host = configs.cortexCppHost;
     const port = configs.cortexCppPort;
-    // Should stop cortex before installing engine
-    const stopCortexSpinner = ora('Stopping cortex...').start();
+    // Should unload engine before installing engine
+    const unloadCortexEngineSpinner = ora('Unloading cortex...').start();
     if (await this.cortexUsecases.healthCheck(host, port)) {
-      await this.cortexUsecases.stopCortex();
+      await this.cortexUsecases.unloadCortexEngine(engine as Engines);
     }
-    stopCortexSpinner.succeed('Cortex stopped');
+
+    unloadCortexEngineSpinner.succeed('Cortex unloaded');
     console.log(`Installing engine ${engine}...`);
 
     await this.cortex.engines.init(engine, params);

cortex-js/src/infrastructure/commanders/ps.command.ts

@@ -12,6 +12,7 @@ import { firstValueFrom } from 'rxjs';
 import { HttpService } from '@nestjs/axios';
 import { CORTEX_CPP_MODELS_URL } from '../constants/cortex';
 import { fileManagerService } from '../services/file-manager/file-manager.service';
+import { getMemoryInformation } from '@/utils/system-resource';
 
 @SubCommand({
   name: 'ps',
@@ -45,9 +46,9 @@ export class PSCommand extends BaseCommand {
             totalVram,
           });
         }
-        const memoryData = await systeminformation.mem();
+        const { total, used } = await getMemoryInformation()
         const memoryUsage = (
-          (memoryData.active / memoryData.total) *
+          (used / total) *
           100
         ).toFixed(2);
         const consumedTable = {

cortex-js/src/infrastructure/constants/cortex.ts

@@ -26,6 +26,11 @@ export const CORTEX_CPP_PROCESS_DESTROY_URL = (
   port: number = defaultCortexCppPort,
 ) => `http://${host}:${port}/processmanager/destroy`;
 
+export const CORTEX_CPP_UNLOAD_ENGINE_URL = (
+  host: string = defaultCortexCppHost,
+  port: number = defaultCortexCppPort,
+) => `http://${host}:${port}/inferences/server/unloadengine`;
+
 export const CORTEX_CPP_HEALTH_Z_URL = (
   host: string = defaultCortexCppHost,
   port: number = defaultCortexCppPort,

cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts

@@ -1,18 +1,20 @@
+import osUtils from 'node-os-utils'
 import {
   ResourceStatus,
   UsedMemInfo,
 } from '@/domain/models/resource.interface';
+import { getMemoryInformation, MemoryInformation } from '@/utils/system-resource';
 import { Injectable } from '@nestjs/common';
 import systemInformation, { Systeminformation } from 'systeminformation';
 
 @Injectable()
 export class ResourcesManagerService {
   async getResourceStatuses(): Promise<ResourceStatus> {
-    const promises = [systemInformation.currentLoad(), systemInformation.mem()];
+    const promises = [systemInformation.currentLoad(), getMemoryInformation()];
     const results = await Promise.all(promises);
 
     const cpuUsage = results[0] as Systeminformation.CurrentLoadData;
-    const memory = results[1] as Systeminformation.MemData;
+    const memory = results[1] as MemoryInformation
     const memInfo: UsedMemInfo = {
       total: memory.total,
       used: memory.used,

cortex-js/src/usecases/cortex/cortex.usecases.ts

@@ -13,11 +13,13 @@ import { fileManagerService } from '@/infrastructure/services/file-manager/file-
 import {
   CORTEX_CPP_HEALTH_Z_URL,
   CORTEX_CPP_PROCESS_DESTROY_URL,
+  CORTEX_CPP_UNLOAD_ENGINE_URL,
   CORTEX_JS_SYSTEM_URL,
   defaultCortexJsHost,
   defaultCortexJsPort,
 } from '@/infrastructure/constants/cortex';
 import { openSync } from 'fs';
+import { Engines } from '@/infrastructure/commanders/types/engine.interface';
 
 @Injectable()
 export class CortexUsecases implements BeforeApplicationShutdown {
@@ -123,6 +125,29 @@ export class CortexUsecases implements BeforeApplicationShutdown {
     }
   }
 
+  /**
+   * Unload the engine
+   */
+  async unloadCortexEngine(engine: Engines): Promise<CortexOperationSuccessfullyDto> {
+    const configs = await fileManagerService.getConfig();
+    try {
+      await firstValueFrom(
+        this.httpService.post(
+          CORTEX_CPP_UNLOAD_ENGINE_URL(
+            configs.cortexCppHost,
+            configs.cortexCppPort,
+          ),
+        ),
+      );
+    } finally {
+      return {
+        message: `${engine} unloaded successfully`,
+        status: 'success',
+      };
+    }
+  }
+
+
   /**
    * Check whether the Cortex CPP is healthy
    * @param host

cortex-js/src/utils/system-resource.ts

@@ -0,0 +1,16 @@
+import osUtils from 'node-os-utils'
+
+export type MemoryInformation = {
+    total: osUtils.MemUsedInfo['totalMemMb'];
+    used: osUtils.MemUsedInfo['usedMemMb'];
+    free: osUtils.MemFreeInfo['freeMemMb']
+}
+
+export const getMemoryInformation = async (): Promise<MemoryInformation> => {
+    const [usedMemoryInfo, freeMemoryInfo] = await Promise.all([osUtils.mem.used(), osUtils.mem.free()])
+    return {
+        total: usedMemoryInfo.totalMemMb,
+        used: usedMemoryInfo.usedMemMb,
+        free: freeMemoryInfo.freeMemMb
+    }
+}