Local deploy

Author: tqchen

scripts/build_site.sh

@@ -12,12 +12,17 @@ echo "Copy local configurations.."
 cp $1 site/llm-chat-config.json
 echo "Copy files..."
 cp web/llm_chat.html site/_includes
-cp web/llm_chat.js site/dist
+cp web/llm_chat.js site/dist/
 cp web/llm_chat.css site/dist/
 
-cp dist/tokenizer.model site/dist
 cp dist/tvmjs_runtime.wasi.js site/dist
 cp dist/tvmjs.bundle.js site/dist
 cp -r dist/sentencepiece site/dist
 
+if [ -d "dist/vicuna-7b/params" ]; then
+    mkdir -p site/dist/vicuna-7b
+    cp -rf dist/models/vicuna-7b/tokenizer.model site/dist/vicuna-7b/
+    cp -rf dist/vicuna-7b/vicuna-7b_webgpu.wasm site/dist/vicuna-7b/
+fi
+
 cd site && jekyll b && cd ..

scripts/local_deploy_site.sh

@@ -5,5 +5,8 @@ scripts/build_site.sh web/local-config.json
 
 echo "symlink parameter location to site.."
 
-# ln -s `pwd`/dist/params site/_site/web-sd-shards-v1-5
+if [ -d "dist/vicuna-7b/params" ]; then
+    rm -rf site/_site/vicuna-7b-params
+    ln -s `pwd`/dist/vicuna-7b/params site/_site/vicuna-7b-params
+fi
 cd site && jekyll serve  --skip-initial-build --host localhost --baseurl /web-llm --port 8888

setup.py

@@ -0,0 +1,28 @@
+from distutils.core import setup
+from setuptools import find_packages
+
+# Note there is no need to setup when
+# running locally in this folder.
+
+setup(
+    name = "web-lmm",
+    version = "0.1.0",
+    license="Apache-2.0",
+    description = "LMM on browser",
+    author = "Web LMM  contributors",
+    url = "https://github.com/mlc-ai/web-llm",
+    keywords = [],
+    packages = find_packages(),
+    install_requires = [
+        "torch",
+        "transformers"
+    ],
+    classifiers = [
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "Topic :: Software Development :: Build Tools",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+  ],
+)

site/.gitignore

@@ -0,0 +1,4 @@
+dist
+llm-chat-config.json
+_includes/stable_diffusion.html
+_site

site/_config.yml

@@ -0,0 +1,41 @@
+
+name: "Web LLM"
+short_name: "WebLLM"
+
+url: https://mlc.ai/web-llm
+
+exclude: [README.md, serve_local.sh]
+
+plugins:
+  - jekyll-remote-theme
+
+remote_theme: mlc-ai/jekyll-theme-mlc
+
+
+# Colorize code snippets with the rogue module if we want to deploy on GH.
+highlighter: rouge
+
+markdown: kramdown
+
+# The path structure for blog posts.
+permalink: /blog/:year/:month/:day/:title.html
+
+# Number of news stories on the front page.
+front_page_news: 8
+
+# Base pathname for links.
+base: '/web-llm'
+
+# make pages for the _projects folder
+collections:
+  projects:
+    output: true
+
+course_title:
+
+# Navigation bar links.
+navigation:
+  - title: Home
+    link: /
+  - title: Github
+    link: https://github.com/mlc-ai/web-llm

site/_includes/head.html

@@ -0,0 +1,4 @@
+<meta http-equiv="origin-trial" content="Agx76XA0ITxMPF0Z8rbbcMllwuxsyp9qdtQaXlLqu1JUrdHB6FPonuyIKJ3CsBREUkeioJck4nn3KO0c0kkwqAMAAABJeyJvcmlnaW4iOiJodHRwOi8vbG9jYWxob3N0Ojg4ODgiLCJmZWF0dXJlIjoiV2ViR1BVIiwiZXhwaXJ5IjoxNjkxNzExOTk5fQ==">
+<meta http-equiv="origin-trial" content="AnmwqQ1dtYDQTYkZ5iMtHdINCaxjE94uWQBKp2yOz1wPTcjSRtOHUGQG+r2BxsEuM0qhxTVnuTjyh31HgTeA8gsAAABZeyJvcmlnaW4iOiJodHRwczovL21sYy5haTo0NDMiLCJmZWF0dXJlIjoiV2ViR1BVIiwiZXhwaXJ5IjoxNjkxNzExOTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZX0=">
+<script src="dist/tvmjs_runtime.wasi.js"></script>
+<script src="dist/tvmjs.bundle.js"></script>

site/_includes/llm_chat.html

@@ -0,0 +1,37 @@
+<script>
+  var tvmjsGlobalEnv = tvmjsGlobalEnv || {};
+</script>
+
+<script type="module">
+  async function getTokenizer(url) {
+    const mod = await import("./dist/sentencepiece/index.js");
+    return await mod.sentencePieceProcessor(url);
+  }
+  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
+</script>
+<script>
+  function handleChatUIInputEnter(event) {
+    if (event.keyCode === 13) {
+      tvmjsGlobalEnv.asyncOnGenerate();
+    }
+  }
+  async function getTokenizer(url) {
+    const mod = await import("./dist/sentencepiece/index.js");
+    return await mod.sentencePieceProcessor(url);
+  }
+  tvmjsGlobalEnv.sentencePieceProcessor = getTokenizer;
+</script>
+
+<script src="dist/llm_chat.js"></script>
+<link href="dist/llm_chat.css" rel="stylesheet" type="text/css"/>
+<h2>Chat</h2>
+
+<div class="chatui">
+  <div class="chatui-chat" id="chatui-chat" height="100">
+  </div>
+
+  <div class="chatui-inputarea">
+    <input id="chatui-input" type="text" class="chatui-input" onkeypress="handleChatUIInputEnter(event)" placeholder="Enter your message...">
+    <button class="chatui-send-btn" onclick="tvmjsGlobalEnv.asyncOnGenerate()">Send</button>
+  </div>
+</div>

site/index.md

@@ -0,0 +1,11 @@
+---
+layout: default
+title: Home
+notitle: true
+---
+
+
+### Demo
+
+{% include llm_chat.html %}
+

web/llm_chat.js

@@ -283,6 +283,7 @@ class LLMChatPipeline {
 
   async evaluate() {
     // run a canonicla evaluateion fo the flow
+    this.#clearKVCache();
     const testPrompt = "The capital of Canada is";
     const ids = await this.tokenizer.encodeIds(testPrompt);
     const inputPromptSize = ids.length;