From 2c42d65b1eb3a07e8f93f900bef0584b2048474a Mon Sep 17 00:00:00 2001 From: irfanpena Date: Wed, 18 Sep 2024 17:12:36 +0700 Subject: [PATCH 1/6] Update sidebar, Installation, Overview --- docs/basic-usage/command-line.md | 4 +- docs/basic-usage/cortexrc.mdx | 12 ++++ docs/basic-usage/overview.mdx | 96 ++++++++++++++++++++++++++++++++ docs/basic-usage/server.mdx | 2 +- docs/cli/cortex.md | 5 +- docs/cli/serve.md | 2 +- docs/installation/linux.mdx | 19 +++---- docs/installation/mac.mdx | 22 +++----- docs/installation/windows.mdx | 29 ++++++---- docs/overview.mdx | 10 ++++ docs/troubleshooting.mdx | 2 +- sidebars.ts | 57 +++++++++++-------- 12 files changed, 194 insertions(+), 66 deletions(-) create mode 100644 docs/basic-usage/cortexrc.mdx create mode 100644 docs/basic-usage/overview.mdx diff --git a/docs/basic-usage/command-line.md b/docs/basic-usage/command-line.md index 05a10fe..2cf8a42 100644 --- a/docs/basic-usage/command-line.md +++ b/docs/basic-usage/command-line.md @@ -11,13 +11,13 @@ slug: "command-line" Cortex has a [Docker](https://docs.docker.com/engine/reference/commandline/cli/) and [Ollama](https://ollama.com/)-inspired [CLI syntax](/docs/cli) for running model operations. ## How It Works -Cortex’s CLI invokes the Cortex Engine’s API, which runs in the background on port `1337`. +Cortex’s CLI invokes the Cortex Engine’s API, which runs in the background on port `3928`. ## Basic Usage ### [Start Cortex Server](/docs/cli) ```bash -# By default the server will be started on port `1337` +# By default the server will be started on port `3928` cortex ``` ### [Run Model](/docs/cli/run) diff --git a/docs/basic-usage/cortexrc.mdx b/docs/basic-usage/cortexrc.mdx new file mode 100644 index 0000000..504a8de --- /dev/null +++ b/docs/basic-usage/cortexrc.mdx @@ -0,0 +1,12 @@ +--- +title: .cortexrc +description: .cortexrc Overview. +slug: "cortexrc" +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +:::warning +🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. +::: \ No newline at end of file diff --git a/docs/basic-usage/overview.mdx b/docs/basic-usage/overview.mdx new file mode 100644 index 0000000..4f9cb4a --- /dev/null +++ b/docs/basic-usage/overview.mdx @@ -0,0 +1,96 @@ +--- +title: API +description: Cortex Server Overview. +slug: "basic-usage" +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +:::warning +🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. +::: + +Cortex has an [API server](https://cortex.so/api-reference) that runs at `localhost:3928`. + + +## Usage +### Start Cortex.cpp Server +```bash +# By default the server will be started on port `3928` +cortex +``` +### Run Model +```bash +# Pull a model +curl --request POST \ + --url http://localhost:3928/v1/models/mistral/pull +# Start the model +curl --request POST \ + --url http://localhost:3928/v1/models/mistral/start \ + --header 'Content-Type: application/json' \ + --data '{ + "prompt_template": "system\n{system_message}\nuser\n{prompt}\nassistant", + "stop": [], + "ngl": 4096, + "ctx_len": 4096, + "cpu_threads": 10, + "n_batch": 2048, + "caching_enabled": true, + "grp_attn_n": 1, + "grp_attn_w": 512, + "mlock": false, + "flash_attn": true, + "cache_type": "f16", + "use_mmap": true, + "engine": "llamacpp" +}' +``` +### Show the Model State +```bash +# Check the model status +curl --request GET \ + --url http://localhost:3928/v1/system/events/model +``` +### Chat with Model +```bash +# Invoke the chat completions endpoint +curl http://localhost:3928/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "", + "messages": [ + { + "role": "user", + "content": "Hello" + }, + ], + "model": "mistral", + "stream": true, + "max_tokens": 1, + "stop": [ + null + ], + "frequency_penalty": 1, + "presence_penalty": 1, + "temperature": 1, + "top_p": 1 +}' +``` +### Stop Model +```bash +# Stop a model +curl --request POST \ + --url http://localhost:3928/v1/models/mistral/stop +``` +### Pull Model +```bash +# Pull a model +curl --request POST \ + --url http://localhost:3928/v1/models/mistral/pull +``` +### Stop Cortex.cpp Server +```bash +# Stop the Cortex.cpp server +cortex stop +``` \ No newline at end of file diff --git a/docs/basic-usage/server.mdx b/docs/basic-usage/server.mdx index 3a74bab..bbf78eb 100644 --- a/docs/basic-usage/server.mdx +++ b/docs/basic-usage/server.mdx @@ -17,7 +17,7 @@ Cortex has an [API server](https://cortex.so/api-reference) that runs at `localh ## Usage ### Start Cortex Server ```bash -# By default the server will be started on port `1337` +# By default the server will be started on port `3928` cortex # Start a server with different port number cortex -a
-p diff --git a/docs/cli/cortex.md b/docs/cli/cortex.md index 7d897bb..9db0893 100644 --- a/docs/cli/cortex.md +++ b/docs/cli/cortex.md @@ -13,7 +13,7 @@ slug: /cli This is the initial command you need to run to start using Cortex.cpp. ::: -This command starts the Cortex.cpp process and the API server, which runs on port `1337` by default. +This command starts the Cortex.cpp API server, which runs on port `3928` by default. ## Usage @@ -50,4 +50,5 @@ For example: - [cortex engines](/docs/cli/engines): Manage Cortex.cpp engines. - [cortex pull|download](/docs/cli/pull): Download a model. - [cortex run](/docs/cli/run): Shortcut to start a model and chat. -- [cortex update](/docs/cli/stop): Update the Cortex.cpp version. +- [cortex update](/docs/cli/update): Update the Cortex.cpp version. +- [cortex stop](/docs/cli/stop): Stop the Cortex.cpp API server. diff --git a/docs/cli/serve.md b/docs/cli/serve.md index 99a8c83..d7193e3 100644 --- a/docs/cli/serve.md +++ b/docs/cli/serve.md @@ -28,7 +28,7 @@ cortex serve [options] stop | Option | Description | Required | Default Value | Example | |----------------------------|-------------------------------------------|----------|---------------|------------------------| | `-a`, `--address
` | Specify the address to use. | No | `localhost` | `-a 192.168.1.1`| -| `-p`, `--port ` | Define the port to serve the application. | No | `1337` | `-p 8080` | +| `-p`, `--port ` | Define the port to serve the application. | No | `3928` | `-p 8080` | | `-d`, `--detach` | Run the server in detached mode. | No | `false` | `-d` | | `-h`, `--help` | Display help information for the command. | No | - | `-h` | diff --git a/docs/installation/linux.mdx b/docs/installation/linux.mdx index 186d0df..bd2be1f 100644 --- a/docs/installation/linux.mdx +++ b/docs/installation/linux.mdx @@ -36,20 +36,17 @@ By default, Cortex.cpp is installed in the following directory: /home//.cortexcpp ``` ## Uninstall Cortex.cpp -```sh -# Uninstall Jan -sudo apt-get remove cortexcpp - -# Remove all user data -rm -rf ~/.cortexcpp +Run the uninstaller script: +```bash +# Stable version +sudo apt remove cortexcpp -# Delete the application data -rm -rf ~/.cortexrc +# Beta version +sudo apt remove cortexcpp-beta +# Nightly version +sudo apt remove cortexcpp-nightly ``` -:::info -Delete the Cortex.cpp data folder located in your home folder. -::: ## Build from Source 1. Clone the Cortex.cpp repository [here](https://github.com/janhq/cortex.cpp). diff --git a/docs/installation/mac.mdx b/docs/installation/mac.mdx index 44a22be..61e1e55 100644 --- a/docs/installation/mac.mdx +++ b/docs/installation/mac.mdx @@ -29,23 +29,19 @@ By default, Cortex.cpp is installed in the following directory: /Users//.cortexcpp ``` ## Uninstall Cortex.cpp -1. Open the Finder menu. -2. Click the Applications option from the sidebar. -3. Find the Jan app or type in the search bar. -4. Use any of these ways to move the Jan app to the Trash: - - Drag the app to the Trash. - - Select the app and choose the Move to Trash option. - - Select the app and press Command-Delete on your keyboard. -5. Use the following command to delete Jan's user data and app cache: +Run the uninstaller script: ```bash -# Remove all user data -rm -rf ~/.cortexcpp +# Stable version +sudo sh cortex-uninstall.sh -# Delete the application data -rm -rf ~/.cortexrc +# Beta version +sudo sh cortex-beta-uninstall.sh + +# Stable version +sudo sh cortex-nightly-uninstall.sh ``` :::info -Delete the Cortex.cpp data folder located in your home folder. +The script requires sudo permission. ::: ## Build from Source 1. Clone the Cortex.cpp repository [here](https://github.com/janhq/cortex.cpp). diff --git a/docs/installation/windows.mdx b/docs/installation/windows.mdx index cdb76bb..0adfce4 100644 --- a/docs/installation/windows.mdx +++ b/docs/installation/windows.mdx @@ -20,9 +20,9 @@ For Windows, Cortex.cpp can be installed in two ways: ## Windows ### Install Cortex.cpp Before installation, make sure that you have met the required [dependencies](#windows) and [hardware](#hardware) to run Cortex. -- Stable: -- Beta: -- Nightly: +- Stable: https://github.com/janhq/cortex.cpp/releases +- Beta: https://github.com/janhq/cortex.cpp/releases +- Nightly: https://github.com/janhq/cortex.cpp/releases #### Data Folder By default, Cortex.cpp is installed in the following directory: ``` @@ -33,10 +33,11 @@ C:\Users\\AppData\Local\cortexcpp C:\Users\\.cortexcpp ``` ### Uninstall Cortex.cpp -To uninstall Cortex.cpp, simply run the `uninstaller.exe` located in the binary data folder after installation. -:::info -Delete the Cortex.cpp data folder located in your home folder. -::: +To uninstall Cortex.cpp: +1. Navigate to Add or Remove program. +2. Search for Cortex.cpp. +3. Click Uninstall. +4. Delete the Cortex.cpp data folder located in your home folder. ## Windows Subsystem Linux :::info Windows Subsystem Linux allows running Linux tools and workflows seamlessly alongside Windows applications. For more information, please see this [article](https://learn.microsoft.com/en-us/windows/wsl/faq). @@ -58,11 +59,17 @@ C:\Users\\AppData\Local\cortexcpp\cortex.exe C:\Users\\.cortexcpp ``` ### Uninstall Cortex.cpp -To uninstall Cortex.cpp, simply run the `uninstaller.exe` located in the binary data folder after installation. -:::info -Delete the Cortex.cpp data folder located in your home folder. -::: +Run the uninstaller script: +```bash +# Stable version +sudo apt remove cortexcpp +# Beta version +sudo apt remove cortexcpp-beta + +# Nightly version +sudo apt remove cortexcpp-nightly +``` ## Build from Source 1. Clone the Cortex.cpp repository [here](https://github.com/janhq/cortex.cpp). diff --git a/docs/overview.mdx b/docs/overview.mdx index 36f36bb..25be18b 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -90,3 +90,13 @@ Cortex.cpp supports the following list of [Built-in Models](/models): :::info Cortex.cpp supports pulling `GGUF` and `ONNX` models from the [Hugging Face Hub](https://huggingface.co). Read how to [Pull models from Hugging Face](/docs/hub/hugging-face/) ::: + +## Cortex.cpp Versions +Cortex.cpp offers three different versions of the app, each serving a unique purpose: +- **Stable**: The official release version of Cortex.cpp, designed for general use with proven stability. +- **Beta**: This version includes upcoming features still in testing, allowing users to try new functionality before the next official release. +- **Nightly**: Automatically built every night, this version includes the latest updates and changes from the engineering team but may be unstable. + +:::info +Each of these versions has a different CLI prefix command. +::: \ No newline at end of file diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index a437ad4..1b38a85 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -129,7 +129,7 @@ To resolve this issue: ## 426 Error Code -This issue occurs when a new Cortex process is started without fully stopping the previous Cortex processes. This causes a heavy load on port `1337` and requires a protocol upgrade. +This issue occurs when a new Cortex process is started without fully stopping the previous Cortex processes. This causes a heavy load on port `3928` and requires a protocol upgrade. To resolve this issue: diff --git a/sidebars.ts b/sidebars.ts index 04490e6..383f2d2 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -50,9 +50,19 @@ const sidebars: SidebarsConfig = { }, ], }, + { + type: "html", + value: "BASIC USAGE", + + className: "sidebar-divider", + }, + { type: "doc", id: "basic-usage/overview", label: "Overview" }, + { type: "doc", id: "basic-usage/cortexrc", label: ".cortexrc" }, + { type: "doc", id: "model-yaml", label: "model.yaml" }, + { type: "doc", id: "data-structures", label: "Data Folder" }, { type: "category", - label: "Integration", + label: "Libraries", link: { type: "generated-index", }, @@ -70,7 +80,21 @@ const sidebars: SidebarsConfig = { }, ], }, - { type: "doc", id: "data-structures", label: "Data Structures" }, + { + type: "category", + label: "Model Sources", + link: { type: "doc", id: "hub/index" }, + collapsed: true, + items: [ + { type: "doc", id: "hub/cortex-hub", label: "Cortex Model Repos" }, + { type: "doc", id: "hub/hugging-face", label: "HuggingFace Repos" }, + { + type: "doc", + id: "hub/nvidia-ngc", + label: "Nvidia Catalog (Coming Soon)", + }, + ], + }, // { // type: "category", // label: "Basic Usage", @@ -105,13 +129,13 @@ const sidebars: SidebarsConfig = { // }, // { type: "doc", id: "telemetry", label: "Telemetry" }, // MODELs - { - type: "html", - value: "MODELS", - className: "sidebar-divider", - }, - { type: "doc", id: "model-overview", label: "Overview" }, - { type: "doc", id: "model-yaml", label: "model.yaml" }, + // { + // type: "html", + // value: "MODELS", + // className: "sidebar-divider", + // }, + // { type: "doc", id: "model-overview", label: "Overview" }, + // { type: "doc", id: "model-yaml", label: "model.yaml" }, // { type: "doc", id: "built-in-models", label: "Built-in Models" }, // { // type: "category", @@ -135,21 +159,6 @@ const sidebars: SidebarsConfig = { // { type: "doc", id: "formats/onnx", label: "ONNX" }, // ], // }, - { - type: "category", - label: "Model Sources", - link: { type: "doc", id: "hub/index" }, - collapsed: true, - items: [ - { type: "doc", id: "hub/cortex-hub", label: "Cortex Model Repos" }, - { type: "doc", id: "hub/hugging-face", label: "HuggingFace Repos" }, - { - type: "doc", - id: "hub/nvidia-ngc", - label: "Nvidia Catalog (Coming Soon)", - }, - ], - }, // BASIC USAGE // { // type: "html", From 1a23cca3b23059bf5851020af427e5c38355a174 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Thu, 19 Sep 2024 13:06:15 +0700 Subject: [PATCH 2/6] Update Quickstart, Installation and CLI to include different versions commands --- docs/basic-usage/overview.mdx | 60 ++++- docs/cli/{chat.md => chat.mdx} | 32 ++- docs/cli/{cortex.md => cortex.mdx} | 32 ++- docs/cli/embeddings.mdx | 36 ++- docs/cli/engines/index.mdx | 150 ++++++++++-- docs/cli/models/{index.md => index.mdx} | 216 +++++++++++++++--- docs/cli/{ps.md => ps.mdx} | 32 ++- docs/cli/{pull.md => pull.mdx} | 32 ++- docs/cli/{run.md => run.mdx} | 34 ++- docs/cli/{stop.md => stop.mdx} | 32 ++- docs/cli/update.mdx | 32 ++- docs/{data-structures.mdx => data-folder.mdx} | 6 +- docs/hub/cortex-hub.mdx | 96 +++++++- docs/hub/hugging-face.mdx | 110 ++++++++- docs/installation/linux.mdx | 37 ++- docs/installation/mac.mdx | 25 +- docs/installation/windows.mdx | 31 ++- docs/quickstart.mdx | 205 ++++++++++++++--- sidebars.ts | 2 +- 19 files changed, 1044 insertions(+), 156 deletions(-) rename docs/cli/{chat.md => chat.mdx} (78%) rename docs/cli/{cortex.md => cortex.mdx} (82%) rename docs/cli/models/{index.md => index.mdx} (70%) rename docs/cli/{ps.md => ps.mdx} (86%) rename docs/cli/{pull.md => pull.mdx} (65%) rename docs/cli/{run.md => run.mdx} (84%) rename docs/cli/{stop.md => stop.mdx} (62%) rename docs/{data-structures.mdx => data-folder.mdx} (96%) diff --git a/docs/basic-usage/overview.mdx b/docs/basic-usage/overview.mdx index 4f9cb4a..d867909 100644 --- a/docs/basic-usage/overview.mdx +++ b/docs/basic-usage/overview.mdx @@ -16,10 +16,32 @@ Cortex has an [API server](https://cortex.so/api-reference) that runs at `localh ## Usage ### Start Cortex.cpp Server -```bash -# By default the server will be started on port `3928` -cortex -``` + + + ```sh + # Stable + cortex + + # Beta + cortex-beta + + # Nightly + cortex-nightly + ``` + + + ```sh + # Stable + cortex.exe + + # Beta + cortex-beta.exe + + # Nightly + cortex-nightly.exe + ``` + + ### Run Model ```bash # Pull a model @@ -90,7 +112,29 @@ curl --request POST \ --url http://localhost:3928/v1/models/mistral/pull ``` ### Stop Cortex.cpp Server -```bash -# Stop the Cortex.cpp server -cortex stop -``` \ No newline at end of file + + + ```sh + # Stable + cortex stop + + # Beta + cortex-beta stop + + # Nightly + cortex-nightly stop + ``` + + + ```sh + # Stable + cortex.exe stop + + # Beta + cortex-beta.exe stop + + # Nightly + cortex-nightly.exe stop + ``` + + \ No newline at end of file diff --git a/docs/cli/chat.md b/docs/cli/chat.mdx similarity index 78% rename from docs/cli/chat.md rename to docs/cli/chat.mdx index 7fa2270..e2fc54f 100644 --- a/docs/cli/chat.md +++ b/docs/cli/chat.mdx @@ -4,6 +4,9 @@ description: Cortex chat command. slug: "chat" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -18,10 +21,33 @@ This CLI command calls the following API endpoint: This command starts a chat session with a specified model, allowing you to interact directly with it through an interactive chat interface. ## Usage + + + ```sh + # Stable + cortex chat [options] + + # Beta + cortex-beta chat [options] + + # Nightly + cortex-nightly chat [options] + ``` + + + ```sh + # Stable + cortex.exe chat [options] + + # Beta + cortex-beta.exe chat [options] + + # Nightly + cortex-nightly.exe chat [options] + ``` + + -```bash -cortex chat [options] -``` :::info This command uses a `model_id` from the model that you have downloaded or available in your file system. ::: diff --git a/docs/cli/cortex.md b/docs/cli/cortex.mdx similarity index 82% rename from docs/cli/cortex.md rename to docs/cli/cortex.mdx index 9db0893..39dd173 100644 --- a/docs/cli/cortex.md +++ b/docs/cli/cortex.mdx @@ -4,6 +4,9 @@ description: Cortex CLI. slug: /cli --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -16,10 +19,33 @@ This is the initial command you need to run to start using Cortex.cpp. This command starts the Cortex.cpp API server, which runs on port `3928` by default. ## Usage + + + ```sh + # Stable + cortex [command] [options] + + # Beta + cortex-beta [command] [options] + + # Nightly + cortex-nightly [command] [options] + ``` + + + ```sh + # Stable + cortex.exe [command] [options] + + # Beta + cortex-beta.exe [command] [options] + + # Nightly + cortex-nightly.exe [command] [options] + ``` + + -```bash -cortex [command] [options] -``` ## Options diff --git a/docs/cli/embeddings.mdx b/docs/cli/embeddings.mdx index 6713326..dfb81d9 100644 --- a/docs/cli/embeddings.mdx +++ b/docs/cli/embeddings.mdx @@ -4,6 +4,9 @@ description: Cortex embeddings command. slug: "embeddings" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -19,14 +22,33 @@ This command creates the embedding vector representing the input text. ## Usage -```bash + + + ```sh + # Stable + cortex embeddings [options] [model_id] [message] + + # Beta + cortex-beta embeddings [options] [model_id] [message] + + # Nightly + cortex-nightly embeddings [options] [model_id] [message] + ``` + + + ```sh + # Stable + cortex.exe embeddings [options] [model_id] [message] + + # Beta + cortex-beta.exe embeddings [options] [model_id] [message] + + # Nightly + cortex-nightly.exe embeddings [options] [model_id] [message] + ``` + + -# With a model started -cortex models start [model_id] -cortex embeddings [options] [message] -# Without any started models -cortex embeddings [options] [model_id] [message] -``` :::info This command uses a `model_id` from the model that you have downloaded or available in your file system. ::: diff --git a/docs/cli/engines/index.mdx b/docs/cli/engines/index.mdx index 53202f4..72513e4 100644 --- a/docs/cli/engines/index.mdx +++ b/docs/cli/engines/index.mdx @@ -2,6 +2,9 @@ title: Cortex Engines --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -13,10 +16,33 @@ This command allows you to manage various engines available within Cortex. **Usage**: + + + ```sh + # Stable + cortex engines [options] [subcommand] + + # Beta + cortex-beta engines [options] [subcommand] + + # Nightly + cortex-nightly engines [options] [subcommand] + ``` + + + ```sh + # Stable + cortex.exe engines [options] [subcommand] + + # Beta + cortex-beta.exe engines [options] [subcommand] + + # Nightly + cortex-nightly.exe engines [options] [subcommand] + ``` + + -```bash -cortex engines [options] [subcommand] -``` **Options**: @@ -35,10 +61,33 @@ This command returns an engine detail defined by an engine `engine_name`. **Usage**: + + + ```sh + # Stable + cortex engines get + + # Beta + cortex-beta engines get + + # Nightly + cortex-nightly engines get + ``` + + + ```sh + # Stable + cortex.exe engines get + + # Beta + cortex-beta.exe engines get + + # Nightly + cortex-nightly.exe engines get + ``` + + -```bash -cortex engines get -``` For example, it returns the following: ```bash β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” @@ -72,10 +121,33 @@ This command lists all the Cortex's engines. **Usage**: + + + ```sh + # Stable + cortex engines list [options] + + # Beta + cortex-beta engines list [options] + + # Nightly + cortex-nightly engines list [options] + ``` + + + ```sh + # Stable + cortex.exe engines list [options] + + # Beta + cortex-beta.exe engines list [options] + + # Nightly + cortex-nightly.exe engines list [options] + ``` + + -```bash -cortex engines list [options] -``` For example, it returns the following: ```bash +---------+---------------------+-------------------------------------------------------------------------------+---------+------------------------------+-----------------+ @@ -110,9 +182,33 @@ This command downloads the required dependencies and installs the engine within - `Tensorrt-llm` **Usage**: -```bash -cortex engines install [options] -``` + + + ```sh + # Stable + cortex engines install [options] + + # Beta + cortex-beta engines install [options] + + # Nightly + cortex-nightly engines install [options] + ``` + + + ```sh + # Stable + cortex.exe engines install [options] + + # Beta + cortex-beta.exe engines install [options] + + # Nightly + cortex-nightly.exe engines install [options] + ``` + + + For Example: ```bash ## Llama.cpp engine @@ -138,9 +234,33 @@ cortex engines install tensorrt-llm This command uninstalls the engine within Cortex. **Usage**: -```bash -cortex engines uninstall [options] -``` + + + ```sh + # Stable + cortex engines uninstall [options] + + # Beta + cortex-beta engines uninstall [options] + + # Nightly + cortex-nightly engines uninstall [options] + ``` + + + ```sh + # Stable + cortex.exe engines uninstall [options] + + # Beta + cortex-beta.exe engines uninstall [options] + + # Nightly + cortex-nightly.exe engines uninstall [options] + ``` + + + For Example: ```bash ## Llama.cpp engine diff --git a/docs/cli/models/index.md b/docs/cli/models/index.mdx similarity index 70% rename from docs/cli/models/index.md rename to docs/cli/models/index.mdx index f1f79e0..afb0385 100644 --- a/docs/cli/models/index.md +++ b/docs/cli/models/index.mdx @@ -2,6 +2,9 @@ title: Cortex Models --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -12,10 +15,32 @@ This command allows you to start, stop, and manage various local or remote model **Usage**: - -```bash -cortex models [options] -``` + + + ```sh + # Stable + cortex models [options] + + # Beta + cortex-beta models [options] + + # Nightly + cortex-nightly models [options] + ``` + + + ```sh + # Stable + cortex.exe models [options] + + # Beta + cortex-beta.exe models [options] + + # Nightly + cortex-nightly.exe models [options] + ``` + + **Options**: @@ -35,10 +60,33 @@ This command returns a model detail defined by a `model_id`. **Usage**: + + + ```sh + # Stable + cortex models get + + # Beta + cortex-beta models get + + # Nightly + cortex-nightly models get + ``` + + + ```sh + # Stable + cortex.exe models get + + # Beta + cortex-beta.exe models get + + # Nightly + cortex-nightly.exe models get + ``` + + -```bash -cortex models get -``` For example, it returns the following: ```bash @@ -116,10 +164,33 @@ This command lists all the downloaded local and remote models. **Usage**: + + + ```sh + # Stable + cortex models list [options] + + # Beta + cortex-beta models list [options] + + # Nightly + cortex-nightly models list [options] + ``` + + + ```sh + # Stable + cortex.exe models list [options] + + # Beta + cortex-beta.exe models list [options] + + # Nightly + cortex-nightly.exe models list [options] + ``` + + -```bash -cortex models list [options] -``` For example, it returns the following: ```bash +---------+----------------+-----------------+---------+ @@ -149,17 +220,32 @@ This command starts a model defined by a `model_id`. **Usage**: - -```bash -# Start a model -cortex models start - -# Start a model with a preset -cortex models start [options] - -# Start with a specified engine -cortex models start [options] :[engine] -``` + + + ```sh + # Stable + cortex models start [options] + + # Beta + cortex-beta models start [options] + + # Nightly + cortex-nightly models start [options] + ``` + + + ```sh + # Stable + cortex.exe models start [options] + + # Beta + cortex-beta.exe models start [options] + + # Nightly + cortex-nightly.exe models start [options] + ``` + + :::info @@ -185,10 +271,33 @@ This command stops a model defined by a `model_id`. **Usage**: + + + ```sh + # Stable + cortex models stop + + # Beta + cortex-beta models stop + + # Nightly + cortex-nightly models stop + ``` + + + ```sh + # Stable + cortex.exe models stop + + # Beta + cortex-beta.exe models stop + + # Nightly + cortex-nightly.exe models stop + ``` + + -```bash -cortex models stop -``` :::info This command uses a `model_id` from the model that you have started before. ::: @@ -209,10 +318,33 @@ This command updates a model configuration defined by a `model_id`. **Usage**: + + + ```sh + # Stable + cortex models update [options] + + # Beta + cortex-beta models update [options] + + # Nightly + cortex-nightly models update [options] + ``` + + + ```sh + # Stable + cortex.exe models update [options] + + # Beta + cortex-beta.exe models update [options] + + # Nightly + cortex-nightly.exe models update [options] + ``` + + -```bash -cortex models update [options] -``` :::info This command uses a `model_id` from the model that you have downloaded or available in your file system. ::: @@ -234,13 +366,37 @@ This command deletes a local model defined by a `model_id`. **Usage**: + + + ```sh + # Stable + cortex models delete + + # Beta + cortex-beta models delete + + # Nightly + cortex-nightly models delete + ``` + + + ```sh + # Stable + cortex.exe models delete + + # Beta + cortex-beta.exe models delete + + # Nightly + cortex-nightly.exe models delete + ``` + + -```bash -cortex models delete -``` :::info This command uses a `model_id` from the model that you have downloaded or available in your file system. ::: + **Options**: | Option | Description | Required | Default value | Example | |---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| diff --git a/docs/cli/ps.md b/docs/cli/ps.mdx similarity index 86% rename from docs/cli/ps.md rename to docs/cli/ps.mdx index cdf22e5..2073871 100644 --- a/docs/cli/ps.md +++ b/docs/cli/ps.mdx @@ -4,6 +4,9 @@ description: Cortex ps command. slug: "ps" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -19,10 +22,33 @@ This command shows the running model and its status. ## Usage + + + ```sh + # Stable + cortex ps [options] + + # Beta + cortex-beta ps [options] + + # Nightly + cortex-nightly ps [options] + ``` + + + ```sh + # Stable + cortex.exe ps [options] + + # Beta + cortex-beta.exe ps [options] + + # Nightly + cortex-nightly.exe ps [options] + ``` + + -```bash -cortex ps [options] -``` For example, it returns the following table: diff --git a/docs/cli/pull.md b/docs/cli/pull.mdx similarity index 65% rename from docs/cli/pull.md rename to docs/cli/pull.mdx index 5dbc478..83ae17e 100644 --- a/docs/cli/pull.md +++ b/docs/cli/pull.mdx @@ -4,6 +4,9 @@ description: Cortex CLI. slug: "pull" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -19,10 +22,33 @@ The downloaded model will be stored in the Cortex folder in your home data direc ## Usage + + + ```sh + # Stable + cortex pull [options] + + # Beta + cortex-beta pull [options] + + # Nightly + cortex-nightly pull [options] + ``` + + + ```sh + # Stable + cortex.exe pull [options] + + # Beta + cortex-beta.exe pull [options] + + # Nightly + cortex-nightly.exe pull [options] + ``` + + -```bash -cortex pull [options] -``` ## Options diff --git a/docs/cli/run.md b/docs/cli/run.mdx similarity index 84% rename from docs/cli/run.md rename to docs/cli/run.mdx index 964ee0b..81c38e3 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.mdx @@ -4,6 +4,9 @@ description: Cortex run command slug: "run" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -20,12 +23,33 @@ This CLI command calls the following API endpoint: This command facilitates the initiation of an interactive chat shell with a specified machine-learning model. ## Usage + + + ```sh + # Stable + cortex [options] :[engine] + + # Beta + cortex-beta [options] :[engine] + + # Nightly + cortex-nightly [options] :[engine] + ``` + + + ```sh + # Stable + cortex.exe [options] :[engine] + + # Beta + cortex-beta.exe [options] :[engine] + + # Nightly + cortex-nightly.exe [options] :[engine] + ``` + + -```bash -cortex run [options] -# With a specified engine -cortex run [options] :[engine] -``` ### `model_id` You can use the [Built-in models](/docs/hub/cortex-hub) or Supported [HuggingFace models](/docs/hub/hugging-face). diff --git a/docs/cli/stop.md b/docs/cli/stop.mdx similarity index 62% rename from docs/cli/stop.md rename to docs/cli/stop.mdx index 3f4aee8..965bd4c 100644 --- a/docs/cli/stop.md +++ b/docs/cli/stop.mdx @@ -4,6 +4,9 @@ description: Cortex stop command. slug: "stop" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -18,10 +21,33 @@ This command stops the API server. ## Usage + + + ```sh + # Stable + cortex stop [options] + + # Beta + cortex-beta stop [options] + + # Nightly + cortex-nightly stop [options] + ``` + + + ```sh + # Stable + cortex.exe stop [options] + + # Beta + cortex-beta.exe stop [options] + + # Nightly + cortex-nightly.exe stop [options] + ``` + + -```bash -cortex stop [options] -``` ## Options diff --git a/docs/cli/update.mdx b/docs/cli/update.mdx index 9355d96..04012f6 100644 --- a/docs/cli/update.mdx +++ b/docs/cli/update.mdx @@ -4,6 +4,9 @@ description: Cortex update command. slug: "update" --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -15,10 +18,33 @@ This command updates Cortex.cpp to the provided version or the latest version. ## Usage + + + ```sh + # Stable + cortex update [options] + + # Beta + cortex-beta update [options] + + # Nightly + cortex-nightly update [options] + ``` + + + ```sh + # Stable + cortex.exe update [options] + + # Beta + cortex-beta.exe update [options] + + # Nightly + cortex-nightly.exe update [options] + ``` + + -```bash -cortex update [options] -``` :::info By default, if no version is specified, Cortex.cpp will be updated to the latest version. ::: diff --git a/docs/data-structures.mdx b/docs/data-folder.mdx similarity index 96% rename from docs/data-structures.mdx rename to docs/data-folder.mdx index 86a1aa7..c2d502f 100644 --- a/docs/data-structures.mdx +++ b/docs/data-folder.mdx @@ -1,7 +1,7 @@ --- -title: Data Structures -description: Cortex.cpp's data structures. -slug: "data-structures" +title: Data Folder +description: Cortex.cpp's data folder. +slug: "data-folder" --- :::warning diff --git a/docs/hub/cortex-hub.mdx b/docs/hub/cortex-hub.mdx index 10e4752..6c9ac50 100644 --- a/docs/hub/cortex-hub.mdx +++ b/docs/hub/cortex-hub.mdx @@ -3,6 +3,9 @@ title: Cortex Model Repos description: Cortex's built-in models are hosted on Huggingface, supporting multi-engine repositories. --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -23,16 +26,89 @@ Cortex Model Repos are hosted on Huggingface for several reasons: ## Usage Download a built-in model from the [Cortex Model Repos](https://huggingface.co/cortexso) using a `model_id`. You can obtain the `model_id` from the Cortex model repository or the model's specific branch. -```bash -# Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main -cortex pull mistral + + + ```sh + # Stable + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex pull mistral + + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex pull mistral:onnx + + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex pull mistral:tensorrt-llm-linux-ada + + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex pull mistral:7b-gguf + + # Beta + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex-beta pull mistral + + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex-beta pull mistral:onnx + + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex-beta pull mistral:tensorrt-llm-linux-ada + + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex-beta pull mistral:7b-gguf + + # Nightly + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex-nightly pull mistral + + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex-nightly pull mistral:onnx + + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex-nightly pull mistral:tensorrt-llm-linux-ada + + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex-nightly pull mistral:7b-gguf + ``` + + + ```sh + # Stable + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex.exe pull mistral + + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex.exe pull mistral:onnx + + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex.exe pull mistral:tensorrt-llm-linux-ada + + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex.exe pull mistral:7b-gguf + + # Beta + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex-beta.exe pull mistral + + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex-beta.exe pull mistral:onnx + + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex-beta.exe pull mistral:tensorrt-llm-linux-ada + + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex-beta.exe pull mistral:7b-gguf + + # Nightly + ## Download the default mistral model from: https://huggingface.co/cortexso/mistral/tree/main + cortex-nightly.exe pull mistral -# Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx -cortex pull mistral:onnx + ## Download an ONNX version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/onnx + cortex-nightly.exe pull mistral:onnx -# Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada -cortex pull mistral:tensorrt-llm-linux-ada + ## Download a TensorRT-LLM version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/tensorrt-llm-linux-ada + cortex-nightly.exe pull mistral:tensorrt-llm-linux-ada -# Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf -cortex pull mistral:7b-gguf -``` \ No newline at end of file + ## Download a 7B version of the mistral model from: https://huggingface.co/cortexso/mistral/tree/7b-gguf + cortex-nightly.exe pull mistral:7b-gguf + ``` + + \ No newline at end of file diff --git a/docs/hub/hugging-face.mdx b/docs/hub/hugging-face.mdx index 8def7fb..a440999 100644 --- a/docs/hub/hugging-face.mdx +++ b/docs/hub/hugging-face.mdx @@ -3,6 +3,9 @@ title: Hugging Face description: Cortex supports all `GGUF` and `ONNX` models available in Huggingface repositories, providing access to a wide range of models. --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + :::warning 🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. ::: @@ -16,27 +19,110 @@ To pull a supported model from HuggingFace, use the format `ORG_ID/MODEL_ID`. ## GGUF ![HF GGUF](/img/docs/gguf.png) To view all available `GGUF` models on HuggingFace, select the `GGUF` tag in the Libraries section. -```bash -# Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization -cortex pull bartowski/Codestral-22B-v0.1-GGUF + + + ```sh + # Stable + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex pull bartowski/Codestral-22B-v0.1-GGUF + + # Pull the gemma-7b model from the google organization + cortex pull google/gemma-7b + + # Beta + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex-beta pull bartowski/Codestral-22B-v0.1-GGUF + + # Pull the gemma-7b model from the google organization + cortex-beta pull google/gemma-7b + + # Nightly + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex-nightly pull bartowski/Codestral-22B-v0.1-GGUF + + # Pull the gemma-7b model from the google organization + cortex-nightly pull google/gemma-7b + ``` + + + ```sh + # Stable + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex.exe pull bartowski/Codestral-22B-v0.1-GGUF + + # Pull the gemma-7b model from the google organization + cortex.exe pull google/gemma-7b + + # Beta + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex-beta.exe pull bartowski/Codestral-22B-v0.1-GGUF -# Pull the gemma-7b model from the google organization -cortex pull google/gemma-7b + # Pull the gemma-7b model from the google organization + cortex-beta.exe pull google/gemma-7b -``` + # Nightly + ## Pull the Codestral-22B-v0.1-GGUF model from the bartowski organization + cortex-nightly.exe pull bartowski/Codestral-22B-v0.1-GGUF + + # Pull the gemma-7b model from the google organization + cortex-nightly.exe pull google/gemma-7b + ``` + + ## ONNX ![HF ONNX](/img/docs/onnx.png) To view all available `ONNX` models on HuggingFace, select the `ONNX` tag in the Libraries section. -```bash -# Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization -cortex pull immich-app/XLM-Roberta-Large-Vit-B-16Plus + + + ```sh + # Stable + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex pull immich-app/XLM-Roberta-Large-Vit-B-16Plus + + # Pull the mt0-base model from the bigscience organization + cortex pull bigscience/mt0-base + + # Beta + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex-beta pull immich-app/XLM-Roberta-Large-Vit-B-16Plus + + # Pull the mt0-base model from the bigscience organization + cortex-beta pull bigscience/mt0-base + + # Nightly + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex-nightly pull immich-app/XLM-Roberta-Large-Vit-B-16Plus + + # Pull the mt0-base model from the bigscience organization + cortex-nightly pull bigscience/mt0-base + ``` + + + ```sh + # Stable + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex.exe pull immich-app/XLM-Roberta-Large-Vit-B-16Plus + + # Pull the mt0-base model from the bigscience organization + cortex.exe pull bigscience/mt0-base + + # Beta + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex-beta.exe pull immich-app/XLM-Roberta-Large-Vit-B-16Plus -# Pull the mt0-base model from the bigscience organization -cortex pull bigscience/mt0-base + # Pull the mt0-base model from the bigscience organization + cortex-beta.exe pull bigscience/mt0-base + # Nightly + ## Pull the XLM-Roberta-Large-Vit-B-16Plus model from the immich-app organization + cortex-nightly.exe pull immich-app/XLM-Roberta-Large-Vit-B-16Plus -``` + # Pull the mt0-base model from the bigscience organization + cortex-nightly.exe pull bigscience/mt0-base + ``` + + ## TensorRT-LLM We are still working to support all available `TensorRT-LLM` models on HuggingFace. For now, Cortex.cpp only supports built-in `TensorRT-LLM` models, which can be downloaded from the [Cortex Model Repos](/docs/hub/cortex-hub). diff --git a/docs/installation/linux.mdx b/docs/installation/linux.mdx index bd2be1f..08cc975 100644 --- a/docs/installation/linux.mdx +++ b/docs/installation/linux.mdx @@ -13,19 +13,34 @@ import Admonition from '@theme/Admonition'; ::: ## Cortex.cpp Installation -Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. -### .deb -- Stable: https://github.com/janhq/cortex.cpp/releases -- Beta: https://github.com/janhq/cortex.cpp/releases -- Nightly: https://github.com/janhq/cortex.cpp/releases - -### .appImage -- Stable: https://github.com/janhq/cortex.cpp/releases -- Beta: https://github.com/janhq/cortex.cpp/releases -- Nightly: https://github.com/janhq/cortex.cpp/releases +Cortex.cpp offers two installer types: +- Network Installers download a minimal system and require an internet connection to fetch packages during installation. +- Local Installers include all necessary packages, enabling offline installation without internet access. :::info -You can also install Cortex.cpp using the Cortex Installer available on [GitHub Releases](https://github.com/janhq/cortex/releases). +Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. ::: +1. Download the Linux installer: + - `.deb`: + - Stable: https://github.com/janhq/cortex.cpp/releases + - Beta: https://github.com/janhq/cortex.cpp/releases + - Nightly: https://github.com/janhq/cortex.cpp/releases + - `.appImage`: + - Stable: https://github.com/janhq/cortex.cpp/releases + - Beta: https://github.com/janhq/cortex.cpp/releases + - Nightly: https://github.com/janhq/cortex.cpp/releases + +2. Ensure that Cortex.cpp is sucessfulyy installed: +```bash +# Stable +cortex + +# Beta +cortex-beta + +# Nightly +cortex-nightly +``` + ### Data Folder By default, Cortex.cpp is installed in the following directory: ``` diff --git a/docs/installation/mac.mdx b/docs/installation/mac.mdx index 61e1e55..a198c2f 100644 --- a/docs/installation/mac.mdx +++ b/docs/installation/mac.mdx @@ -12,13 +12,28 @@ import TabItem from '@theme/TabItem'; ::: ## Cortex.cpp Installation -Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. -- Stable: https://github.com/janhq/cortex.cpp/releases -- Beta: https://github.com/janhq/cortex.cpp/releases -- Nightly: https://github.com/janhq/cortex.cpp/releases +Cortex.cpp offers two installer types: +- Network Installers download a minimal system and require an internet connection to fetch packages during installation. +- Local Installers include all necessary packages, enabling offline installation without internet access. :::info -You can also install Cortex.cpp using the Cortex.cpp Installer available on [GitHub Releases](https://github.com/janhq/cortex/releases). +Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. ::: +1. Download the MacOs installer: + - Stable: https://github.com/janhq/cortex.cpp/releases + - Beta: https://github.com/janhq/cortex.cpp/releases + - Nightly: https://github.com/janhq/cortex.cpp/releases +2. Ensure that Cortex.cpp is sucessfulyy installed: +```bash +# Stable +cortex + +# Beta +cortex-beta + +# Nightly +cortex-nightly +``` + ### Data Folder By default, Cortex.cpp is installed in the following directory: ``` diff --git a/docs/installation/windows.mdx b/docs/installation/windows.mdx index 0adfce4..ac2d3c3 100644 --- a/docs/installation/windows.mdx +++ b/docs/installation/windows.mdx @@ -19,7 +19,12 @@ For Windows, Cortex.cpp can be installed in two ways: ## Windows ### Install Cortex.cpp -Before installation, make sure that you have met the required [dependencies](#windows) and [hardware](#hardware) to run Cortex. +Cortex.cpp offers two installer types: +- Network Installers download a minimal system and require an internet connection to fetch packages during installation. +- Local Installers include all necessary packages, enabling offline installation without internet access. +:::info +Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. +::: - Stable: https://github.com/janhq/cortex.cpp/releases - Beta: https://github.com/janhq/cortex.cpp/releases - Nightly: https://github.com/janhq/cortex.cpp/releases @@ -43,11 +48,27 @@ To uninstall Cortex.cpp: Windows Subsystem Linux allows running Linux tools and workflows seamlessly alongside Windows applications. For more information, please see this [article](https://learn.microsoft.com/en-us/windows/wsl/faq). ::: ### Install Cortex.cpp -Before installation, make sure that you have met the required [dependencies](#windows-subsystem-for-linux) and [hardware](#hardware) to run Cortex. +Cortex.cpp offers two installer types: +- Network Installers download a minimal system and require an internet connection to fetch packages during installation. +- Local Installers include all necessary packages, enabling offline installation without internet access. +:::info +Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. +::: +1. Download the Windows installer: + - Stable: https://github.com/janhq/cortex.cpp/releases + - Beta: https://github.com/janhq/cortex.cpp/releases + - Nightly: https://github.com/janhq/cortex.cpp/releases +2. Ensure that Cortex.cpp is sucessfulyy installed: +```bash +# Stable +cortex.exe -- Stable: https://github.com/janhq/cortex.cpp/releases -- Beta: https://github.com/janhq/cortex.cpp/releases -- Nightly: https://github.com/janhq/cortex.cpp/releases +# Beta +cortex-beta.exe + +# Nightly +cortex-nightly.exe +``` #### Data Folder By default, Cortex.cpp is installed in the following directory: diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 5c59052..0e77ce2 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -21,23 +21,64 @@ To install Cortex, download the installer for your operating system from the fol - [Linux (Fedora)](https://github.com/janhq/cortex.cpp/releases) ## Start Cortex.cpp Processes and API Server This command starts the Cortex.cpp API server at `localhost:3928`. -```sh -cortex -``` + + + ```sh + # Stable + cortex + + # Beta + cortex-beta + + # Nightly + cortex-nightly + ``` + + + ```sh + # Stable + cortex.exe + + # Beta + cortex-beta.exe + + # Nightly + cortex-nightly.exe + ``` + + ## Run a Model -This command downloads the default `gguf` model format from the [Cortex Hub](https://huggingface.co/cortexso) and starts the model. -```sh -cortex run mistral -``` +This command downloads the default `gguf` model format from the [Cortex Hub](https://huggingface.co/cortexso), starts the model, and chat with the model. + + + ```sh + # Stable + cortex run mistral + + # Beta + cortex-beta run mistral + + # Nightly + cortex-nightly run mistral + ``` + + + ```sh + # Stable + cortex.exe run mistral + + # Beta + cortex-beta.exe run mistral + + # Nightly + cortex-nightly.exe run mistral + ``` + + :::info All model files are stored in the `~users/cortex/models` folder. ::: ## Using the Model -### CLI -```sh -# CLI -cortex chat mistral -``` ### API ```curl curl http://localhost:3928/v1/chat/completions \ @@ -86,25 +127,137 @@ completion = client.chat.completions.create( ``` ## Stop a Model This command stops the running model. - ```bash -cortex models stop -``` + + + ```sh + # Stable + cortex models stop tinyllama + + # Beta + cortex-beta models stop tinyllama + + # Nightly + cortex-nightly models stop tinyllama + ``` + + + ```sh + # Stable + cortex.exe models stop tinyllama + + # Beta + cortex-beta.exe models stop tinyllama + + # Nightly + cortex-nightly.exe models stop tinyllama + ``` + + ## Show the System State This command displays the running model and the hardware system status. - ```bash -cortex ps -``` + + + ```sh + # Stable + cortex ps + + # Beta + cortex-beta ps + + # Nightly + cortex-nightly ps + ``` + + + ```sh + # Stable + cortex.exe ps + + # Beta + cortex-beta.exe ps + + # Nightly + cortex-nightly.exe ps + ``` + + ## Run Different Model Variants -```bash -# Run HuggingFace model with HuggingFace Repo -cortex run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + + ```sh + # Stable + ## Run HuggingFace model with HuggingFace Repo + cortex run TheBloke/Mistral-7B-Instruct-v0.2-GGUF -# Run Mistral in ONNX format -cortex run mistral:onnx + # Run Mistral in ONNX format + cortex run mistral:onnx -# Run Mistral in TensorRT-LLM format -cortex run mistral:tensorrt-llm -``` + # Run Mistral in TensorRT-LLM format + cortex run mistral:tensorrt-llm + + # Beta + ## Run HuggingFace model with HuggingFace Repo + cortex-beta run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + # Run Mistral in ONNX format + cortex-beta run mistral:onnx + + # Run Mistral in TensorRT-LLM format + cortex-beta run mistral:tensorrt-llm + + # Nightly + ## Run HuggingFace model with HuggingFace Repo + cortex-nightly run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + # Run Mistral in ONNX format + cortex-nightly run mistral:onnx + + # Run Mistral in TensorRT-LLM format + cortex-nightly run mistral:tensorrt-llm + ``` + + + ```sh + # Stable + ## Run HuggingFace model with HuggingFace Repo + cortex.exe run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + # Run Mistral in ONNX format + cortex.exe run mistral:onnx + + # Run Mistral in TensorRT-LLM format + cortex.exe run mistral:tensorrt-llm + + # Beta + ## Run HuggingFace model with HuggingFace Repo + cortex-beta.exe run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + # Run Mistral in ONNX format + cortex-beta.exe run mistral:onnx + + # Run Mistral in TensorRT-LLM format + cortex-beta.exe run mistral:tensorrt-llm + + # Nightly + ## Run HuggingFace model with HuggingFace Repo + cortex-nightly.exe run TheBloke/Mistral-7B-Instruct-v0.2-GGUF + + # Run Mistral in ONNX format + cortex-nightly.exe run mistral:onnx + + # Run Mistral in TensorRT-LLM format + cortex-nightly.exe run mistral:tensorrt-llm + ``` + + + +## What's Next? +Now that Cortex.cpp is set up, here are the next steps to explore: + +1. Adjust the folder path and configuration using the [`.cortexrc`](/docs/basic-usage/cortexrc) file. +2. Explore the Cortex.cpp [data folder](/docs/data-folder) to understand how it stores data. +3. Learn about the structure of the [`model.yaml`](/docs/model-yaml) file in Cortex.cpp. +4. Integrate Cortex.cpp [libraries](/docs/category/libraries) seamlessly into your Python or JavaScript applications. :::info diff --git a/sidebars.ts b/sidebars.ts index 383f2d2..ee4aefc 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -59,7 +59,7 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "basic-usage/overview", label: "Overview" }, { type: "doc", id: "basic-usage/cortexrc", label: ".cortexrc" }, { type: "doc", id: "model-yaml", label: "model.yaml" }, - { type: "doc", id: "data-structures", label: "Data Folder" }, + { type: "doc", id: "data-folder", label: "Data Folder" }, { type: "category", label: "Libraries", From f0e3f05aab99b6e676aaee5056ec967a6973f256 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Mon, 23 Sep 2024 08:08:31 +0700 Subject: [PATCH 3/6] Update windows uninstallation steps --- docs/installation/windows.mdx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/installation/windows.mdx b/docs/installation/windows.mdx index ac2d3c3..11dce33 100644 --- a/docs/installation/windows.mdx +++ b/docs/installation/windows.mdx @@ -39,10 +39,8 @@ C:\Users\\.cortexcpp ``` ### Uninstall Cortex.cpp To uninstall Cortex.cpp: -1. Navigate to Add or Remove program. -2. Search for Cortex.cpp. -3. Click Uninstall. -4. Delete the Cortex.cpp data folder located in your home folder. +1. Navigate to **Add or Remove program**. +2. Search for Cortex.cpp and click **Uninstall**. ## Windows Subsystem Linux :::info Windows Subsystem Linux allows running Linux tools and workflows seamlessly alongside Windows applications. For more information, please see this [article](https://learn.microsoft.com/en-us/windows/wsl/faq). From 952ce6b655004d5c87edcc85e0f591eb076c6878 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Mon, 23 Sep 2024 15:02:18 +0700 Subject: [PATCH 4/6] Update the CLI commands --- docs/cli/chat.mdx | 3 + docs/cli/cortex.mdx | 34 +++----- docs/cli/embeddings.mdx | 4 +- docs/cli/engines/index.mdx | 15 ++++ docs/cli/models/index.mdx | 155 ++++++++++++++++++++++++++++++++++++- docs/cli/ps.mdx | 3 + docs/cli/pull.mdx | 3 + docs/cli/run.mdx | 3 + docs/cli/start.mdx | 60 ++++++++++++++ docs/cli/stop.mdx | 3 + docs/cli/update.mdx | 3 + sidebars.ts | 1 + 12 files changed, 259 insertions(+), 28 deletions(-) create mode 100644 docs/cli/start.mdx diff --git a/docs/cli/chat.mdx b/docs/cli/chat.mdx index e2fc54f..a5f1770 100644 --- a/docs/cli/chat.mdx +++ b/docs/cli/chat.mdx @@ -21,6 +21,9 @@ This CLI command calls the following API endpoint: This command starts a chat session with a specified model, allowing you to interact directly with it through an interactive chat interface. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/cortex.mdx b/docs/cli/cortex.mdx index 39dd173..517494c 100644 --- a/docs/cli/cortex.mdx +++ b/docs/cli/cortex.mdx @@ -12,53 +12,40 @@ import TabItem from "@theme/TabItem"; ::: # Cortex -:::info -This is the initial command you need to run to start using Cortex.cpp. -::: - -This command starts the Cortex.cpp API server, which runs on port `3928` by default. +This command list all the available commands within the Cortex.cpp commands. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh # Stable - cortex [command] [options] + cortex # Beta - cortex-beta [command] [options] + cortex-beta # Nightly - cortex-nightly [command] [options] + cortex-nightly ``` ```sh # Stable - cortex.exe [command] [options] + cortex.exe # Beta - cortex-beta.exe [command] [options] + cortex-beta.exe # Nightly - cortex-nightly.exe [command] [options] + cortex-nightly.exe ``` -## Options - -| Option | Description | Required | Default value | Example | -| ---------------------------- | ----------------------------------------- | -------- | ------------- | ----------------------------- | -| `-v`, `--version` | Show version. | No | - | `-v` | -| `-h`, `--help` | Display help information for the command. | No | - | `-h` | -| `--verbose` | Show the detailed command logs | No | - | `--verbose` | - - - - ## Command Chaining Cortex CLI's command chaining support allows multiple commands to be executed in sequence with a simplified syntax. @@ -77,4 +64,5 @@ For example: - [cortex pull|download](/docs/cli/pull): Download a model. - [cortex run](/docs/cli/run): Shortcut to start a model and chat. - [cortex update](/docs/cli/update): Update the Cortex.cpp version. +- [cortex start](/docs/cli/start): Start the Cortex.cpp API server. - [cortex stop](/docs/cli/stop): Stop the Cortex.cpp API server. diff --git a/docs/cli/embeddings.mdx b/docs/cli/embeddings.mdx index dfb81d9..d43a26d 100644 --- a/docs/cli/embeddings.mdx +++ b/docs/cli/embeddings.mdx @@ -21,7 +21,9 @@ This command creates the embedding vector representing the input text. ## Usage - +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/engines/index.mdx b/docs/cli/engines/index.mdx index 72513e4..6b07fa8 100644 --- a/docs/cli/engines/index.mdx +++ b/docs/cli/engines/index.mdx @@ -16,6 +16,9 @@ This command allows you to manage various engines available within Cortex. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -61,6 +64,9 @@ This command returns an engine detail defined by an engine `engine_name`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -121,6 +127,9 @@ This command lists all the Cortex's engines. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -182,6 +191,9 @@ This command downloads the required dependencies and installs the engine within - `Tensorrt-llm` **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -234,6 +246,9 @@ cortex engines install tensorrt-llm This command uninstalls the engine within Cortex. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/models/index.mdx b/docs/cli/models/index.mdx index afb0385..6ebf52b 100644 --- a/docs/cli/models/index.mdx +++ b/docs/cli/models/index.mdx @@ -15,17 +15,20 @@ This command allows you to start, stop, and manage various local or remote model **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh # Stable - cortex models [options] + cortex models [options] [subcommand] # Beta - cortex-beta models [options] + cortex-beta models [options] [subcommand] # Nightly - cortex-nightly models [options] + cortex-nightly models [options] [subcommand] ``` @@ -60,6 +63,9 @@ This command returns a model detail defined by a `model_id`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -164,6 +170,9 @@ This command lists all the downloaded local and remote models. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -220,6 +229,9 @@ This command starts a model defined by a `model_id`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -271,6 +283,9 @@ This command stops a model defined by a `model_id`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -318,6 +333,9 @@ This command updates a model configuration defined by a `model_id`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -366,6 +384,9 @@ This command deletes a local model defined by a `model_id`. **Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh @@ -401,4 +422,130 @@ This command uses a `model_id` from the model that you have downloaded or availa | Option | Description | Required | Default value | Example | |---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| | `model_id` | The identifier of the model you want to delete. | Yes | - | `mistral` | -| `-h`, `--help` | Display help for command. | No | - | `-h` | \ No newline at end of file +| `-h`, `--help` | Display help for command. | No | - | `-h` | + +## `cortex models alias` +This command adds an alias to a local model that function the same as `model_id`. + + +**Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: + + + ```sh + # Stable + cortex models alias --model_id --alias + + # Beta + cortex-beta models alias --model_id --alias + + # Nightly + cortex-nightly models alias --model_id --alias + ``` + + + ```sh + # Stable + cortex.exe models alias --model_id --alias + + # Beta + cortex-beta.exe models alias --model_id --alias + + # Nightly + cortex-nightly.exe models alias --model_id --alias + ``` + + + + +**Options**: +| Option | Description | Required | Default value | Example | +|---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| +| `--model_id` | The identifier of the model. | Yes | - | `mistral` | +| `-alias` | The new identifier for the model. | Yes | - | `mistral_2` | + +## `cortex models update` +This command updates the `model.yaml` file of a local model. + + +**Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: + + + ```sh + # Stable + cortex models update [options] + + # Beta + cortex-beta models update [options] + + # Nightly + cortex-nightly models update [options] + ``` + + + ```sh + # Stable + cortex.exe models update [options] + + # Beta + cortex-beta.exe models update [options] + + # Nightly + cortex-nightly.exe models update [options] + ``` + + + + +**Options**: +| Option | Description | Required | Default value | Example | +|---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| +| `-h`, `--help` | Display help for command. | No | - | `-h` | + +## `cortex models import` +This command imports the local model using the model's `gguf` file. + + +**Usage**: +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: + + + ```sh + # Stable + cortex models import --model_id --model_path + + # Beta + cortex-beta models import --model_id --model_path + + # Nightly + cortex-nightly models import --model_id --model_path + ``` + + + ```sh + # Stable + cortex.exe models import --model_id --model_path + + # Beta + cortex-beta.exe models import --model_id --model_path + + # Nightly + cortex-nightly.exe models import --model_id --model_path + ``` + + + + +**Options**: +| Option | Description | Required | Default value | Example | +|---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| +| `-h`, `--help` | Display help for command. | No | - | `-h` | +| `--model_id` | The identifier of the model. | Yes | - | `mistral` | +| `--model_path` | The path of the model source file. | Yes | - | `/path/to/your/model.gguf` | \ No newline at end of file diff --git a/docs/cli/ps.mdx b/docs/cli/ps.mdx index 2073871..cbe09fa 100644 --- a/docs/cli/ps.mdx +++ b/docs/cli/ps.mdx @@ -22,6 +22,9 @@ This command shows the running model and its status. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/pull.mdx b/docs/cli/pull.mdx index 83ae17e..df1f391 100644 --- a/docs/cli/pull.mdx +++ b/docs/cli/pull.mdx @@ -22,6 +22,9 @@ The downloaded model will be stored in the Cortex folder in your home data direc ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/run.mdx b/docs/cli/run.mdx index 81c38e3..88911ba 100644 --- a/docs/cli/run.mdx +++ b/docs/cli/run.mdx @@ -23,6 +23,9 @@ This CLI command calls the following API endpoint: This command facilitates the initiation of an interactive chat shell with a specified machine-learning model. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/start.mdx b/docs/cli/start.mdx new file mode 100644 index 0000000..87075d1 --- /dev/null +++ b/docs/cli/start.mdx @@ -0,0 +1,60 @@ +--- +title: Cortex Start +description: Cortex CLI. +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +:::warning +🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. +::: + +# Start +:::info +This is the initial command you need to run to start using Cortex.cpp. +::: + +This command start the Cortex.cpp's API server processes. + +## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: + + + ```sh + # Stable + cortex start [options] + + # Beta + cortex-beta start [options] + + # Nightly + cortex-nightly start [options] + ``` + + + ```sh + # Stable + cortex.exe start [options] + + # Beta + cortex-beta.exe start [options] + + # Nightly + cortex-nightly.exe start [options] + ``` + + + + +## Options + +| Option | Description | Required | Default value | Example | +| ---------------------------- | ----------------------------------------- | -------- | ------------- | ----------------------------- | +| `-h`, `--help` | Display help information for the command. | No | - | `-h` | +| `-p`, `--port ` | Port to serve the application. | No | - | `-p 3928` | + + + diff --git a/docs/cli/stop.mdx b/docs/cli/stop.mdx index 965bd4c..48c4eec 100644 --- a/docs/cli/stop.mdx +++ b/docs/cli/stop.mdx @@ -21,6 +21,9 @@ This command stops the API server. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/docs/cli/update.mdx b/docs/cli/update.mdx index 04012f6..f54d554 100644 --- a/docs/cli/update.mdx +++ b/docs/cli/update.mdx @@ -18,6 +18,9 @@ This command updates Cortex.cpp to the provided version or the latest version. ## Usage +:::info +You can use the `--verbose` flag to display more detailed output of the internal processes. To apply this flag, use the following format: `cortex --verbose [subcommand]`. +::: ```sh diff --git a/sidebars.ts b/sidebars.ts index ee4aefc..43c91dc 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -184,6 +184,7 @@ const sidebars: SidebarsConfig = { className: "sidebar-divider", }, { type: "doc", id: "cli/cortex", label: "cortex" }, + { type: "doc", id: "cli/start", label: "cortex start" }, { type: "doc", id: "cli/chat", label: "cortex chat" }, { type: "doc", id: "cli/embeddings", label: "cortex embeddings" }, // { type: "doc", id: "cli/presets", label: "cortex presets" }, From 458d099f627f44b5d926090979218f67e015ca8e Mon Sep 17 00:00:00 2001 From: irfanpena Date: Tue, 24 Sep 2024 14:16:48 +0700 Subject: [PATCH 5/6] Update model update options --- docs/cli/models/index.mdx | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/cli/models/index.mdx b/docs/cli/models/index.mdx index 6ebf52b..9ac9b62 100644 --- a/docs/cli/models/index.mdx +++ b/docs/cli/models/index.mdx @@ -506,6 +506,50 @@ You can use the `--verbose` flag to display more detailed output of the internal | Option | Description | Required | Default value | Example | |---------------------------|-----------------------------------------------------------------------------|----------|----------------------|------------------------| | `-h`, `--help` | Display help for command. | No | - | `-h` | +| `--model_id REQUIRED` | Unique identifier for the model. | Yes | - | `--model_id my_model` | +| `--name` | Name of the model. | No | - | `--name "GPT Model"` | +| `--model` | Model type or architecture. | No | - | `--model GPT-4` | +| `--version` | Version of the model to use. | No | - | `--version 1.2.0` | +| `--stop` | Stop token to terminate generation. | No | - | `--stop ""` | +| `--top_p` | Sampling parameter for nucleus sampling. | No | - | `--top_p 0.9` | +| `--temperature` | Controls randomness in generation. | No | - | `--temperature 0.8` | +| `--frequency_penalty` | Penalizes repeated tokens based on frequency. | No | - | `--frequency_penalty 0.5` | +| `--presence_penalty` | Penalizes repeated tokens based on presence. | No | `0.0` | `--presence_penalty 0.6` | +| `--max_tokens` | Maximum number of tokens to generate. | No | - | `--max_tokens 1500` | +| `--stream` | Stream output tokens as they are generated. | No | `false` | `--stream true` | +| `--ngl` | Number of generations in parallel. | No | - | `--ngl 4` | +| `--ctx_len` | Maximum context length in tokens. | No | - | `--ctx_len 1024` | +| `--engine` | Compute engine for running the model. | No | - | `--engine CUDA` | +| `--prompt_template` | Template for the prompt structure. | No | - | `--prompt_template "###"` | +| `--system_template` | Template for system-level instructions. | No | - | `--system_template "SYSTEM"` | +| `--user_template` | Template for user inputs. | No | - | `--user_template "USER"` | +| `--ai_template` | Template for AI responses. | No | - | `--ai_template "ASSISTANT"` | +| `--os` | Operating system environment. | No | - | `--os Ubuntu` | +| `--gpu_arch` | GPU architecture specification. | No | - | `--gpu_arch A100` | +| `--quantization_method` | Quantization method for model weights. | No | - | `--quantization_method int8` | +| `--precision` | Floating point precision for computations. | No | `float32` | `--precision float16` | +| `--tp` | Tensor parallelism. | No | - | `--tp 4` | +| `--trtllm_version` | Version of the TRTLLM library. | No | - | `--trtllm_version 2.0` | +| `--text_model` | The model used for text generation. | No | - | `--text_model llama2` | +| `--files` | File path or resources associated with the model. | No | - | `--files config.json` | +| `--created` | Creation date of the model. | No | - | `--created 2024-01-01` | +| `--object` | The object type (e.g., model or file). | No | - | `--object model` | +| `--owned_by` | The owner or creator of the model. | No | - | `--owned_by "Company"` | +| `--seed` | Seed for random number generation. | No | - | `--seed 42` | +| `--dynatemp_range` | Range for dynamic temperature scaling. | No | - | `--dynatemp_range 0.7-1.0` | +| `--dynatemp_exponent` | Exponent for dynamic temperature scaling. | No | - | `--dynatemp_exponent 1.2` | +| `--top_k` | Top K sampling to limit token selection. | No | - | `--top_k 50` | +| `--min_p` | Minimum probability threshold for tokens. | No | - | `--min_p 0.1` | +| `--tfs_z` | Token frequency selection scaling factor. | No | - | `--tfs_z 0.5` | +| `--typ_p` | Typicality-based token selection probability. | No | - | `--typ_p 0.9` | +| `--repeat_last_n` | Number of last tokens to consider for repetition penalty. | No | - | `--repeat_last_n 64` | +| `--repeat_penalty` | Penalty for repeating tokens. | No | - | `--repeat_penalty 1.2` | +| `--mirostat` | Mirostat sampling method for stable generation. | No | - | `--mirostat 1` | +| `--mirostat_tau` | Target entropy for Mirostat. | No | - | `--mirostat_tau 5.0` | +| `--mirostat_eta` | Learning rate for Mirostat. | No | - | `--mirostat_eta 0.1` | +| `--penalize_nl` | Penalize new lines in generation. | No | `false` | `--penalize_nl true` | +| `--ignore_eos` | Ignore the end of sequence token. | No | `false` | `--ignore_eos true` | +| `--n_probs` | Number of probability outputs to return. | No | - | `--n_probs 5` | ## `cortex models import` This command imports the local model using the model's `gguf` file. From 8b9042cf7cabd4e246db37e766d44b70b7bdf03a Mon Sep 17 00:00:00 2001 From: irfanpena Date: Wed, 25 Sep 2024 15:05:55 +0700 Subject: [PATCH 6/6] Update docs per new behaviour --- docs/basic-usage/overview.mdx | 16 ++++++++-------- docs/cli/chat.mdx | 12 ++++++------ docs/quickstart.mdx | 24 ++++++++++++------------ 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/basic-usage/overview.mdx b/docs/basic-usage/overview.mdx index d867909..24b3c97 100644 --- a/docs/basic-usage/overview.mdx +++ b/docs/basic-usage/overview.mdx @@ -1,6 +1,6 @@ --- -title: API -description: Cortex Server Overview. +title: Overview +description: Overview. slug: "basic-usage" --- @@ -20,25 +20,25 @@ Cortex has an [API server](https://cortex.so/api-reference) that runs at `localh ```sh # Stable - cortex + cortex start # Beta - cortex-beta + cortex-beta start # Nightly - cortex-nightly + cortex-nightly start ``` ```sh # Stable - cortex.exe + cortex.exe start # Beta - cortex-beta.exe + cortex-beta.exe start # Nightly - cortex-nightly.exe + cortex-nightly.exe start ``` diff --git a/docs/cli/chat.mdx b/docs/cli/chat.mdx index a5f1770..6615ac0 100644 --- a/docs/cli/chat.mdx +++ b/docs/cli/chat.mdx @@ -28,25 +28,25 @@ You can use the `--verbose` flag to display more detailed output of the internal ```sh # Stable - cortex chat [options] + cortex chat [options] -m # Beta - cortex-beta chat [options] + cortex-beta chat [options] -m # Nightly - cortex-nightly chat [options] + cortex-nightly chat [options] -m ``` ```sh # Stable - cortex.exe chat [options] + cortex.exe chat [options] -m # Beta - cortex-beta.exe chat [options] + cortex-beta.exe chat [options] -m # Nightly - cortex-nightly.exe chat [options] + cortex-nightly.exe chat [options] -m ``` diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 0e77ce2..ee3b35d 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -25,25 +25,25 @@ This command starts the Cortex.cpp API server at `localhost:3928`. ```sh # Stable - cortex + cortex start # Beta - cortex-beta + cortex-beta start # Nightly - cortex-nightly + cortex-nightly start ``` ```sh # Stable - cortex.exe + cortex.exe start # Beta - cortex-beta.exe + cortex-beta.exe start # Nightly - cortex-nightly.exe + cortex-nightly.exe start ``` @@ -131,25 +131,25 @@ This command stops the running model. ```sh # Stable - cortex models stop tinyllama + cortex models stop mistral # Beta - cortex-beta models stop tinyllama + cortex-beta models stop mistral # Nightly - cortex-nightly models stop tinyllama + cortex-nightly models stop mistral ``` ```sh # Stable - cortex.exe models stop tinyllama + cortex.exe models stop mistral # Beta - cortex-beta.exe models stop tinyllama + cortex-beta.exe models stop mistral # Nightly - cortex-nightly.exe models stop tinyllama + cortex-nightly.exe models stop mistral ```