Skip to content

Update LlamaLib to v1.2.3 (llama.cpp b4688) #312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 19, 2025
Merged

Update LlamaLib to v1.2.3 (llama.cpp b4688) #312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6844480
remove support for CUDA 11.7.1
amakropoulos Feb 13, 2025
cdb08d1
implement DeepSeek chat templates
amakropoulos Feb 14, 2025
d453b3c
add Qwen 2.5 and DeepSeek R1 Distil models
amakropoulos Feb 14, 2025
f7bd3c7
bump LlamaLib to v1.2.3
amakropoulos Feb 14, 2025
da45a25
fix DeepSeek chat templates
amakropoulos Feb 18, 2025
e2bfb16
adapt unit tests
amakropoulos Feb 18, 2025
d1a146f
adapt CUDA full tests
amakropoulos Feb 19, 2025
bb5d6e3
adapt tests for windows
amakropoulos Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 117 additions & 40 deletions Runtime/LLMChatTemplates.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
/// @file
/// @brief File implementing the chat templates.
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;

namespace LLMUnity
{
Expand Down Expand Up @@ -44,6 +46,9 @@ static ChatTemplate()
new Phi3_5Template(),
new Phi3Template(),
new Phi2Template(),
new DeepSeekR1Template(),
new DeepSeekV3Template(),
new DeepSeekV2Template(),
new VicunaTemplate(),
new ZephyrTemplate(),
};
Expand All @@ -60,12 +65,12 @@ static ChatTemplate()
templatesDescription[template.GetDescription()] = template.GetName();
foreach (string match in template.GetNameMatches())
{
if (modelTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"{match} already in modelTemplates");
if (modelTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"Name for {template.GetName()} already in modelTemplates");
modelTemplates[match] = template.GetName();
}
foreach (string match in template.GetChatTemplateMatches())
{
if (chatTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"{match} already in chatTemplates");
if (chatTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"Chat template for {template.GetName()} already in chatTemplates");
chatTemplates[match] = template.GetName();
}
}
Expand Down Expand Up @@ -164,13 +169,33 @@ public static ChatTemplate GetTemplate(string template)
protected virtual string RequestSuffix() { return ""; }
protected virtual string PairSuffix() { return ""; }

protected virtual bool SystemPromptSupported() { return true; }

/// <summary> Constructs the prompt using the template based on a list of ChatMessages </summary>
/// <param name="messages"> list of ChatMessages e.g. the LLMCharacter chat </param>
/// <param name="AIName"> the AI name </param>
/// <param name="endWithPrefix"> whether to end the prompt with the AI prefix </param>
/// <returns>prompt</returns>
public virtual string ComputePrompt(List<ChatMessage> messages, string playerName, string AIName, bool endWithPrefix = true)
public virtual string ComputePrompt(List<ChatMessage> chatMessages, string playerName, string AIName, bool endWithPrefix = true)
{
List<ChatMessage> messages = chatMessages;
if (!SystemPromptSupported())
{
if (chatMessages[0].role == "system")
{
string firstUserMessage = chatMessages[0].content;
int newStart = 1;
if (chatMessages.Count > 1)
{
if (firstUserMessage != "") firstUserMessage += "\n\n";
firstUserMessage += chatMessages[1].content;
newStart = 2;
}
messages = new List<ChatMessage>(){new ChatMessage { role = playerName, content = firstUserMessage }};
messages.AddRange(chatMessages.GetRange(newStart, chatMessages.Count - newStart));
}
}

string chatPrompt = PromptPrefix();
int start = 0;
if (messages[0].role == "system")
Expand Down Expand Up @@ -352,24 +377,7 @@ public class GemmaTemplate : ChatTemplate
protected override string PlayerPrefix(string playerName) { return "<start_of_turn>" + playerName + "\n"; }
protected override string AIPrefix(string AIName) { return "<start_of_turn>" + AIName + "\n"; }

public override string ComputePrompt(List<ChatMessage> messages, string playerName, string AIName, bool endWithPrefix = true)
{
List<ChatMessage> messagesSystemPrompt = messages;
if (messages[0].role == "system")
{
string firstUserMessage = messages[0].content;
int start = 1;
if (messages.Count > 1)
{
if (firstUserMessage != "") firstUserMessage += "\n\n";
firstUserMessage += messages[1].content;
start = 2;
}
messagesSystemPrompt = new List<ChatMessage>(){new ChatMessage { role = playerName, content = firstUserMessage }};
messagesSystemPrompt.AddRange(messages.GetRange(start, messages.Count - start));
}
return base.ComputePrompt(messagesSystemPrompt, playerName, AIName, endWithPrefix);
}
protected override bool SystemPromptSupported() { return false; }

public override string[] GetStop(string playerName, string AIName)
{
Expand Down Expand Up @@ -461,25 +469,7 @@ public class Phi3Template : ChatTemplate
protected override string RequestSuffix() { return "<|end|>\n"; }
protected override string PairSuffix() { return "<|end|>\n"; }


public override string ComputePrompt(List<ChatMessage> messages, string playerName, string AIName, bool endWithPrefix = true)
{
List<ChatMessage> messagesSystemPrompt = messages;
if (messages[0].role == "system")
{
string firstUserMessage = messages[0].content;
int start = 1;
if (messages.Count > 1)
{
if (firstUserMessage != "") firstUserMessage += "\n\n";
firstUserMessage += messages[1].content;
start = 2;
}
messagesSystemPrompt = new List<ChatMessage>(){new ChatMessage { role = "user", content = firstUserMessage }};
messagesSystemPrompt.AddRange(messages.GetRange(start, messages.Count - start));
}
return base.ComputePrompt(messagesSystemPrompt, playerName, AIName, endWithPrefix);
}
protected override bool SystemPromptSupported() { return false; }

public override string[] GetStop(string playerName, string AIName)
{
Expand Down Expand Up @@ -534,4 +524,91 @@ public override string[] GetStop(string playerName, string AIName)
return AddStopNewlines(new string[] { $"<|user|>", $"<|assistant|>" });
}
}

/// @ingroup template
/// <summary>
/// Class implementing the DeepSeek V2 template
/// </summary>
public class DeepSeekV2Template : ChatTemplate
{
public override string GetName() { return "deepseek-v2"; }
public override string GetDescription() { return "deepseek-v2"; }
public override string[] GetNameMatches() { return new string[] {"deepseek-v2", "deepseek-llm"}; }
public override string[] GetChatTemplateMatches() { return new string[] {"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\n\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"}; }

protected override string PrefixMessageSeparator() { return " "; }
protected override string PromptPrefix() { return "<｜begin▁of▁sentence｜>"; }
protected override string PlayerPrefix(string playerName) { return "User:"; }
protected override string AIPrefix(string AIName) { return "Assistant:"; }
protected override string PairSuffix() { return "<｜end▁of▁sentence｜>"; }
protected override string RequestSuffix() { return "\n\n"; }
protected override string SystemSuffix() { return "\n\n"; }

// protected override bool SystemPromptSupported() { return false; }

public override string[] GetStop(string playerName, string AIName)
{
return AddStopNewlines(new string[] { "<｜end▁of▁sentence｜>", "User:", "Assistant:" });
}
}

/// @ingroup template
/// <summary>
/// Class implementing the DeepSeek V3 template
/// </summary>
public class DeepSeekV3Template : DeepSeekV2Template
{
public override string GetName() { return "deepseek-v3"; }
public override string GetDescription() { return "deepseek-v3"; }
public override string[] GetNameMatches() { return new string[] {"deepseek-v2.5", "deepseek-v3"}; }
public override string[] GetChatTemplateMatches()
{
return new string[]
{
"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}",
"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}"
};
}

protected override string PrefixMessageSeparator() { return ""; }
protected override string PlayerPrefix(string playerName) { return "<｜User｜>"; }
protected override string AIPrefix(string AIName) { return "<｜Assistant｜>"; }
protected override string RequestSuffix() { return ""; }

public override string[] GetStop(string playerName, string AIName)
{
return AddStopNewlines(new string[] { "<｜end▁of▁sentence｜>", "<｜User｜>", "<｜Assistant｜>" });
}
}

/// @ingroup template
/// <summary>
/// Class implementing the DeepSeek R1 template
/// </summary>
public class DeepSeekR1Template : DeepSeekV3Template
{
public override string GetName() { return "deepseek-r1"; }
public override string GetDescription() { return "deepseek-r1"; }
public override string[] GetNameMatches() { return new string[] {"deepseek-r1"}; }
public override string[] GetChatTemplateMatches()
{
return new string[]
{
"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\\n\\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}"
};
}

public override string ComputePrompt(List<ChatMessage> chatMessages, string playerName, string AIName, bool endWithPrefix = true)
{
string prompt = base.ComputePrompt(chatMessages, playerName, AIName, endWithPrefix);
if (endWithPrefix) prompt += "<think>\n\n</think>\n\n";
return prompt;
}

public override string[] GetStop(string playerName, string AIName)
{
return AddStopNewlines(new string[] { "<｜end▁of▁sentence｜>", "<｜User｜>", "<｜Assistant｜>", "</think>" });
}
}
}
Loading
Loading