Skip to content

Commit 54e75f5

Browse files
Ankur-singhAnkur-singh
authored
WebLLM chat demo featuring in-browser inference (#182)
* add webllm chat application * move webLLM sample to Web-Apps directory
1 parent 507651a commit 54e75f5

File tree

10 files changed

+850
-0
lines changed

10 files changed

+850
-0
lines changed

‎Web-Apps/WebLLM/01_completion.html‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head></head>
4+
<body>
5+
<h1 style="text-align: center;">WebLLM Chat Bot</h1>
6+
<p style="text-align: center;">Completion with the LLM model</p>
7+
<p style="text-align: center;">Open the console to see the response</p>
8+
<!-- External JavaScript File -->
9+
<script type="module">
10+
// Source: https://webllm.mlc.ai/docs/user/basic_usage.html#chat-completion
11+
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
12+
13+
// Initialize with a progress callback
14+
const initProgressCallback = (progress) => {
15+
console.log("Model loading progress:", progress);
16+
};
17+
18+
19+
// Using CreateMLCEngine
20+
const engine = await webllm.CreateMLCEngine("Llama-3.2-1B-Instruct-q4f32_1-MLC", { initProgressCallback });
21+
22+
const messages = [
23+
{ role: "system", content: "You are a helpful AI assistant." },
24+
{ role: "user", content: "Hello!" }
25+
];
26+
27+
const reply = await engine.chat.completions.create({
28+
messages,
29+
});
30+
31+
console.log(reply.choices[0].message);
32+
console.log(reply.usage);
33+
</script>
34+
</body>
35+
</html>

‎Web-Apps/WebLLM/02_model_select.html‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head></head>
4+
<body>
5+
<h1 style="text-align: center;">WebLLM Chat Bot</h1>
6+
<p style="text-align: center;">Model Selection</p>
7+
<p style="text-align: center;">Open the console to see the response</p>
8+
<!-- External JavaScript File -->
9+
<script type="module">
10+
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
11+
12+
const availableModels = webllm.prebuiltAppConfig.model_list;
13+
console.log("Available models:", availableModels);
14+
</script>
15+
</body>
16+
</html>

‎Web-Apps/WebLLM/03_streaming.html‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head></head>
4+
<body>
5+
<h1 style="text-align: center;">WebLLM Chat Bot</h1>
6+
<p style="text-align: center;">Streaming the LLM output</p>
7+
<p style="text-align: center;">Open the console to see the response</p>
8+
<!-- External JavaScript File -->
9+
<script type="module">
10+
// Source: https://webllm.mlc.ai/docs/user/basic_usage.html#streaming-chat-completion
11+
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
12+
13+
// Initialize with a progress callback
14+
const initProgressCallback = (progress) => {
15+
console.log("Model loading progress:", progress);
16+
};
17+
18+
19+
// Using CreateMLCEngine
20+
const engine = await webllm.CreateMLCEngine("Llama-3.2-1B-Instruct-q4f32_1-MLC", { initProgressCallback });
21+
22+
const messages = [
23+
{ role: "system", content: "You are a helpful AI assistant." },
24+
{ role: "user", content: "Hello!" }
25+
];
26+
27+
// Chunks is an AsyncGenerator object
28+
const chunks = await engine.chat.completions.create({
29+
messages,
30+
temperature: 1,
31+
stream: true, // <-- Enable streaming
32+
stream_options: { include_usage: true },
33+
});
34+
35+
let reply = "";
36+
for await (const chunk of chunks) {
37+
reply += chunk.choices[0]?.delta.content || "";
38+
console.log(reply);
39+
if (chunk.usage) {
40+
console.log(chunk.usage); // only last chunk has usage
41+
}
42+
}
43+
44+
const fullReply = await engine.getMessage();
45+
console.log(fullReply);
46+
</script>
47+
</body>
48+
</html>

‎Web-Apps/WebLLM/11_basicUI.html‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
6+
<title>Chatbot in the Browser</title>
7+
</head>
8+
<body>
9+
<div id="loading-progress"></div>
10+
<div style="margin: 10px;">
11+
<textarea id="chat-input" placeholder="Type a message..." rows="2"></textarea>
12+
<button id="send-btn">Send</button>
13+
</div>
14+
<div id="message-container"></div>
15+
<!-- External JavaScript File -->
16+
<script type="module">
17+
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
18+
19+
// DOM Selectors
20+
const messageContainer = document.getElementById("message-container");
21+
const sendButton = document.getElementById("send-btn");
22+
const chatInput = document.getElementById("chat-input");
23+
const loadingProgress = document.getElementById("loading-progress");
24+
25+
// Initialize with a progress callback
26+
const initProgressCallback = (progress) => {
27+
loadingProgress.textContent = `Model loading progress: ${progress.text}`;
28+
console.log("Model loading progress:", progress);
29+
};
30+
31+
// Create engine instance
32+
const engine = await webllm.CreateMLCEngine("Llama-3.2-1B-Instruct-q4f32_1-MLC", { initProgressCallback });
33+
34+
// Update chat UI with new messages
35+
function updateChatUI(message, role) {
36+
const messageDiv = document.createElement("div");
37+
messageDiv.textContent = `${role}: ${message}`;
38+
messageContainer.appendChild(messageDiv);
39+
}
40+
41+
// Stream message generation from the engine
42+
async function streamGeneration(messages) {
43+
const chunks = await engine.chat.completions.create({
44+
messages,
45+
temperature: 1,
46+
stream: true,
47+
stream_options: { include_usage: true },
48+
});
49+
50+
let reply = "AI: ";
51+
for await (const chunk of chunks) {
52+
reply += chunk.choices[0]?.delta.content || "";
53+
messageContainer.innerHTML = reply;
54+
if (chunk.usage) {
55+
console.log(chunk.usage); // only last chunk has usage
56+
}
57+
}
58+
59+
const fullReply = await engine.getMessage();
60+
console.log(fullReply);
61+
}
62+
63+
// Send a new message
64+
function onSend() {
65+
if (chatInput.value.trim() !== "") {
66+
sendButton.disabled = true;
67+
68+
// User message
69+
const userMessage = { role: "user", content: chatInput.value };
70+
71+
// Clear input field
72+
chatInput.value = "";
73+
74+
// Generate response
75+
const messages = [
76+
{ role: "system", content: "You are a helpful AI assistant." },
77+
userMessage
78+
];
79+
streamGeneration(messages).then(() => {
80+
sendButton.disabled = false;
81+
});
82+
}
83+
}
84+
85+
// Event listener for send button
86+
sendButton.addEventListener("click", onSend);
87+
</script>
88+
</body>
89+
</html>

‎Web-Apps/WebLLM/12_generationUI.html‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
6+
<script src="https://cdn.tailwindcss.com"></script>
7+
<title>Chatbot in the Browser</title>
8+
</head>
9+
<body class="bg-gray-100 font-sans">
10+
<div class="max-w-4xl mx-auto p-6">
11+
<!-- Header -->
12+
<h1 class="text-center p-6 text-3xl font-bold text-blue-600">LLM Generation in the Browser</h1>
13+
14+
<!-- Model Selection Section -->
15+
<div class="flex justify-center items-center space-x-4 my-4">
16+
<label for="model-selection" class="text-s font-semibold text-gray-900">Model</label>
17+
<select name="model-selection" id="model-selection" class="p-2 rounded-lg border border-gray-300 text-gray-700 w-full">
18+
<option value="">Please select</option>
19+
<!-- Add options here -->
20+
</select>
21+
<button id="download" class="rounded-lg bg-indigo-600 px-6 py-2 text-sm font-medium text-white hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-600 focus:ring-opacity-50">
22+
Download
23+
</button>
24+
</div>
25+
26+
27+
<!-- Loading Progress -->
28+
<div id="loading-progress" class="hidden mt-4 text-center text-sm text-gray-600">Downloading model...></div>
29+
30+
<!-- Chat Container -->
31+
<div class="my-3 w-full flex items-center space-x-2">
32+
<!-- Chat Input -->
33+
<textarea id="chat-input" class="flex-grow p-2 border border-gray-300 rounded-lg" placeholder="Type a message..." rows="2"></textarea>
34+
<button id="send-btn" class="rounded border border-indigo-600 bg-indigo-600 px-4 py-2 text-sm font-medium text-white hover:bg-transparent hover:text-indigo-600 focus:outline-none focus:ring active:text-indigo-500 disabled:bg-gray-400 disabled:cursor-not-allowed" disabled>Send</button>
35+
</div>
36+
<div id="message-container"></div>
37+
</div>
38+
</div>
39+
40+
<!-- External JavaScript File -->
41+
<script type="module">
42+
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
43+
44+
// DOM Selectors
45+
const messageContainer = document.getElementById("message-container");
46+
const sendButton = document.getElementById("send-btn");
47+
const chatInput = document.getElementById("chat-input");
48+
const loadingProgress = document.getElementById("loading-progress");
49+
const modelSelection = document.getElementById("model-selection");
50+
const downloadButton = document.getElementById("download");
51+
52+
// Update model options
53+
let selectedModel = "SmolLM2-135M-Instruct-q0f16-MLC";
54+
const availableModels = webllm.prebuiltAppConfig.model_list;
55+
56+
function updateModelOptions() {
57+
availableModels.forEach((model) => {
58+
if (model.low_resource_required) {
59+
const option = document.createElement("option");
60+
option.value = model.model_id;
61+
option.textContent = `${model.model_id} (${model.vram_required_MB} MB)`;
62+
modelSelection.appendChild(option);
63+
}
64+
});
65+
modelSelection.value = selectedModel;
66+
}
67+
68+
updateModelOptions();
69+
70+
71+
// Initialize with a progress callback
72+
const initProgressCallback = (progress) => {
73+
loadingProgress.textContent = `Model loading progress: ${progress.text}`;
74+
console.log("Model loading progress:", progress);
75+
};
76+
77+
// Create WebLLM engine instance
78+
const engine = new webllm.MLCEngine({
79+
initProgressCallback: initProgressCallback,
80+
});
81+
82+
// Initialize engine with selected model
83+
async function initializeWebLLMEngine() {
84+
loadingProgress.classList.remove("hidden");
85+
selectedModel = modelSelection.value;
86+
const config = { temperature: 1.0, top_p: 1 };
87+
await engine.reload(selectedModel, config);
88+
}
89+
90+
downloadButton.addEventListener("click", function () {
91+
initializeWebLLMEngine().then(() => {
92+
sendButton.disabled = false;
93+
});
94+
});
95+
96+
// Update chat UI with new messages
97+
function updateChatUI(message, role) {
98+
const messageDiv = document.createElement("div");
99+
messageDiv.textContent = `${role}: ${message}`;
100+
messageContainer.appendChild(messageDiv);
101+
}
102+
103+
// Stream message generation from the engine
104+
async function streamGeneration(messages) {
105+
const chunks = await engine.chat.completions.create({
106+
messages,
107+
temperature: 1,
108+
stream: true,
109+
stream_options: { include_usage: true },
110+
});
111+
112+
let reply = "AI: ";
113+
for await (const chunk of chunks) {
114+
reply += chunk.choices[0]?.delta.content || "";
115+
messageContainer.innerHTML = reply;
116+
if (chunk.usage) {
117+
console.log(chunk.usage); // only last chunk has usage
118+
}
119+
}
120+
121+
const fullReply = await engine.getMessage();
122+
console.log(fullReply);
123+
}
124+
125+
// Send a new message
126+
function onSend() {
127+
if (chatInput.value.trim() !== "") {
128+
sendButton.disabled = true;
129+
130+
// User message
131+
const userMessage = { role: "user", content: chatInput.value };
132+
133+
// Clear input field
134+
chatInput.value = "";
135+
136+
// Generate response
137+
const messages = [
138+
{ role: "system", content: "You are a helpful AI assistant." },
139+
userMessage
140+
];
141+
streamGeneration(messages).then(() => {
142+
sendButton.disabled = false;
143+
});
144+
}
145+
}
146+
147+
// Event listener for send button
148+
sendButton.addEventListener("click", onSend);
149+
</script>
150+
</body>
151+
</html>

0 commit comments

Comments
 (0)