diff --git a/.github/doxygen/Doxyfile b/.github/doxygen/Doxyfile index d66f248b..f0f3fa3a 100644 --- a/.github/doxygen/Doxyfile +++ b/.github/doxygen/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "LLM for Unity" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = v2.0.2 +PROJECT_NUMBER = v2.0.3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/CHANGELOG.md b/CHANGELOG.md index 20399de8..d45dc1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## v2.0.3 +#### 🚀 Features + +- Add LLM selector in Inspector mode (PR: #182) +- Allow to save chat history at custom path (PR: #179) +- Use asynchronous startup by default (PR: #186) +- Assign LLM if not set according to the scene and hierarchy (PR: #187) +- Allow to set log level (PR: #189) +- Allow to add callback functions for error messages (PR: #190) +- Allow to set a LLM base prompt for all LLMCharacter objects (PR: #192) + +#### 🐛 Fixes + +- set higher priority for mac build with Accelerate than without (PR: #180) +- Fix duplicate bos warning + + ## v2.0.2 #### 🐛 Fixes diff --git a/CHANGELOG.release.md b/CHANGELOG.release.md index e4f88118..f692fe7f 100644 --- a/CHANGELOG.release.md +++ b/CHANGELOG.release.md @@ -1,6 +1,15 @@ -### 🐛 Fixes +### 🚀 Features + +- Add LLM selector in Inspector mode (PR: #182) +- Allow to save chat history at custom path (PR: #179) +- Use asynchronous startup by default (PR: #186) +- Assign LLM if not set according to the scene and hierarchy (PR: #187) +- Allow to set log level (PR: #189) +- Allow to add callback functions for error messages (PR: #190) +- Allow to set a LLM base prompt for all LLMCharacter objects (PR: #192) -- Fix bugs in chat completion (PR: #176) -- Call DontDestroyOnLoad on root to remove warning (PR: #174) +### 🐛 Fixes +- set higher priority for mac build with Accelerate than without (PR: #180) +- Fix duplicate bos warning diff --git a/Editor/LLMCharacterEditor.cs b/Editor/LLMCharacterEditor.cs index f50738cd..34de32a5 100644 --- a/Editor/LLMCharacterEditor.cs +++ b/Editor/LLMCharacterEditor.cs @@ -13,18 +13,6 @@ protected override Type[] GetPropertyTypes() return new Type[] { typeof(LLMCharacter) }; } - public void AddClientSettings(SerializedObject llmScriptSO) - { - List attributeClasses = new List(){typeof(LocalRemoteAttribute)}; - attributeClasses.Add(llmScriptSO.FindProperty("remote").boolValue ? typeof(RemoteAttribute) : typeof(LocalAttribute)); - attributeClasses.Add(typeof(LLMAttribute)); - if (llmScriptSO.FindProperty("advancedOptions").boolValue) - { - attributeClasses.Add(typeof(LLMAdvancedAttribute)); - } - ShowPropertiesOfClass("Setup Settings", llmScriptSO, attributeClasses, true); - } - public void AddModelSettings(SerializedObject llmScriptSO, LLMCharacter llmCharacterScript) { EditorGUILayout.LabelField("Model Settings", EditorStyles.boldLabel); @@ -51,30 +39,20 @@ public void AddModelSettings(SerializedObject llmScriptSO, LLMCharacter llmChara } } - public void AddChatSettings(SerializedObject llmScriptSO) - { - ShowPropertiesOfClass("Chat Settings", llmScriptSO, new List { typeof(ChatAttribute) }, true); - } - public override void OnInspectorGUI() { LLMCharacter llmScript = (LLMCharacter)target; SerializedObject llmScriptSO = new SerializedObject(llmScript); - llmScriptSO.Update(); - GUI.enabled = false; - AddScript(llmScriptSO); - GUI.enabled = true; - EditorGUI.BeginChangeCheck(); + OnInspectorGUIStart(llmScriptSO); AddOptionsToggles(llmScriptSO); - AddClientSettings(llmScriptSO); + + AddSetupSettings(llmScriptSO); AddChatSettings(llmScriptSO); + Space(); AddModelSettings(llmScriptSO, llmScript); - if (EditorGUI.EndChangeCheck()) - Repaint(); - - llmScriptSO.ApplyModifiedProperties(); + OnInspectorGUIEnd(llmScriptSO); } } diff --git a/Editor/LLMEditor.cs b/Editor/LLMEditor.cs index 1229e817..39e30dc9 100644 --- a/Editor/LLMEditor.cs +++ b/Editor/LLMEditor.cs @@ -14,12 +14,6 @@ protected override Type[] GetPropertyTypes() return new Type[] { typeof(LLM) }; } - public void AddServerLoadersSettings(SerializedObject llmScriptSO, LLM llmScript) - { - EditorGUILayout.LabelField("Setup Settings", EditorStyles.boldLabel); - AddServerSettings(llmScriptSO); - } - public void AddModelLoadersSettings(SerializedObject llmScriptSO, LLM llmScript) { EditorGUILayout.LabelField("Model Settings", EditorStyles.boldLabel); @@ -100,24 +94,6 @@ public void AddModelSettings(SerializedObject llmScriptSO) Space(); } - public void AddServerSettings(SerializedObject llmScriptSO) - { - List attributeClasses = new List(){typeof(LocalRemoteAttribute)}; - attributeClasses.Add(llmScriptSO.FindProperty("remote").boolValue ? typeof(RemoteAttribute) : typeof(LocalAttribute)); - attributeClasses.Add(typeof(LLMAttribute)); - if (llmScriptSO.FindProperty("advancedOptions").boolValue) - { - attributeClasses.Add(typeof(LLMAdvancedAttribute)); - } - ShowPropertiesOfClass("", llmScriptSO, attributeClasses, true); - Space(); - } - - public void AddChatSettings(SerializedObject llmScriptSO) - { - ShowPropertiesOfClass("Chat Settings", llmScriptSO, new List { typeof(ChatAttribute) }, false); - } - void ShowProgress(float progress, string progressText) { if (progress != 1) EditorGUI.ProgressBar(EditorGUILayout.GetControlRect(), progress, progressText); @@ -126,15 +102,9 @@ void ShowProgress(float progress, string progressText) public override void OnInspectorGUI() { LLM llmScript = (LLM)target; - // LLM llmScript = (LLM)target; SerializedObject llmScriptSO = new SerializedObject(llmScript); - llmScriptSO.Update(); - - GUI.enabled = false; - AddScript(llmScriptSO); - GUI.enabled = true; - EditorGUI.BeginChangeCheck(); + OnInspectorGUIStart(llmScriptSO); ShowProgress(LLMUnitySetup.libraryProgress, "Setup Library"); ShowProgress(llmScript.modelProgress, "Model Downloading"); @@ -142,15 +112,12 @@ public override void OnInspectorGUI() GUI.enabled = LLMUnitySetup.libraryProgress == 1 && llmScript.modelProgress == 1 && llmScript.modelCopyProgress == 1; AddOptionsToggles(llmScriptSO); - AddServerLoadersSettings(llmScriptSO, llmScript); + AddSetupSettings(llmScriptSO); AddModelLoadersSettings(llmScriptSO, llmScript); GUI.enabled = true; AddChatSettings(llmScriptSO); - if (EditorGUI.EndChangeCheck()) - Repaint(); - - llmScriptSO.ApplyModifiedProperties(); + OnInspectorGUIEnd(llmScriptSO); } } } diff --git a/Editor/PropertyEditor.cs b/Editor/PropertyEditor.cs index e596e533..7229804a 100644 --- a/Editor/PropertyEditor.cs +++ b/Editor/PropertyEditor.cs @@ -29,8 +29,31 @@ public void AddOptionsToggle(SerializedObject llmScriptSO, string propertyName, } } + public void AddSetupSettings(SerializedObject llmScriptSO) + { + List attributeClasses = new List(){typeof(LocalRemoteAttribute)}; + attributeClasses.Add(llmScriptSO.FindProperty("remote").boolValue ? typeof(RemoteAttribute) : typeof(LocalAttribute)); + attributeClasses.Add(typeof(LLMAttribute)); + if (llmScriptSO.FindProperty("advancedOptions").boolValue) + { + attributeClasses.Add(typeof(LLMAdvancedAttribute)); + } + ShowPropertiesOfClass("Setup Settings", llmScriptSO, attributeClasses, true); + } + + public void AddChatSettings(SerializedObject llmScriptSO) + { + List attributeClasses = new List(){typeof(ChatAttribute)}; + if (llmScriptSO.FindProperty("advancedOptions").boolValue) + { + attributeClasses.Add(typeof(ChatAdvancedAttribute)); + } + ShowPropertiesOfClass("Chat Settings", llmScriptSO, attributeClasses, false); + } + public void AddOptionsToggles(SerializedObject llmScriptSO) { + LLMUnitySetup.SetDebugMode((LLMUnitySetup.DebugModeType)EditorGUILayout.EnumPopup("Log Level", LLMUnitySetup.DebugMode)); EditorGUILayout.BeginHorizontal(); AddOptionsToggle(llmScriptSO, "advancedOptions", "Advanced Options"); EditorGUILayout.EndHorizontal(); @@ -127,5 +150,22 @@ public Attribute GetPropertyAttribute(SerializedProperty prop, Type attributeCla } return null; } + + public void OnInspectorGUIStart(SerializedObject scriptSO) + { + scriptSO.Update(); + GUI.enabled = false; + AddScript(scriptSO); + GUI.enabled = true; + EditorGUI.BeginChangeCheck(); + } + + public void OnInspectorGUIEnd(SerializedObject scriptSO) + { + if (EditorGUI.EndChangeCheck()) + Repaint(); + + scriptSO.ApplyModifiedProperties(); + } } } diff --git a/README.md b/README.md index 70206057..16bf1921 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ LLM for Unity is built on top of the awesome [llama.cpp](https://github.com/gger At a glance  •  How to help  •  -Games using LLM for Unity  •  +Games using LLM for Unity  •  Setup  •  How to use  •  Examples  •  @@ -309,6 +309,7 @@ HuggingFace models can be converted to gguf with this [online converter](https:/ ### LLM Settings - `Show/Hide Advanced Options` Toggle to show/hide advanced options from below +- `Log Level` select how verbose the log messages are #### 💻 Setup Settings @@ -345,10 +346,17 @@ If the user's GPU is not supported, the LLM will fall back to the CPU +#### 🗨️ Chat Settings +-
Advanced options + +- `Base Prompt` a common base prompt to use across all LLMCharacter objects using the LLM + +
### LLMCharacter Settings - `Show/Hide Advanced Options` Toggle to show/hide advanced options from below +- `Log Level` select how verbose the log messages are #### 💻 Setup Settings
diff --git a/Runtime/LLM.cs b/Runtime/LLM.cs index f1f8f9e0..74e51e33 100644 --- a/Runtime/LLM.cs +++ b/Runtime/LLM.cs @@ -65,7 +65,7 @@ public class LLM : MonoBehaviour /// } /// \endcode /// - [LLMAdvanced] public bool asynchronousStartup = false; + [LLMAdvanced] public bool asynchronousStartup = true; /// select to not destroy the LLM GameObject when loading a new Scene. [LLMAdvanced] public bool dontDestroyOnLoad = true; /// the path of the model being used (relative to the Assets/StreamingAssets folder). @@ -79,13 +79,15 @@ public class LLM : MonoBehaviour [ModelAdvanced] public int contextSize = 0; /// Batch size for prompt processing. [ModelAdvanced] public int batchSize = 512; + /// a base prompt to use as a base for all LLMCharacter objects + [TextArea(5, 10), ChatAdvanced] public string basePrompt = ""; + /// Boolean set to true if the server has started and is ready to receive requests, false otherwise. public bool started { get; protected set; } = false; /// Boolean set to true if the server has failed to start. public bool failed { get; protected set; } = false; /// \cond HIDE - public string slotSaveDir; public int SelectedModel = 0; [HideInInspector] public float modelProgress = 1; [HideInInspector] public float modelCopyProgress = 1; @@ -174,13 +176,13 @@ protected virtual string GetLlamaccpArguments() // Start the LLM server in a cross-platform way if (model == "") { - Debug.LogError("No model file provided!"); + LLMUnitySetup.LogError("No model file provided!"); return null; } string modelPath = LLMUnitySetup.GetAssetPath(model); if (!File.Exists(modelPath)) { - Debug.LogError($"File {modelPath} not found!"); + LLMUnitySetup.LogError($"File {modelPath} not found!"); return null; } string loraPath = ""; @@ -189,7 +191,7 @@ protected virtual string GetLlamaccpArguments() loraPath = LLMUnitySetup.GetAssetPath(lora); if (!File.Exists(loraPath)) { - Debug.LogError($"File {loraPath} not found!"); + LLMUnitySetup.LogError($"File {loraPath} not found!"); return null; } } @@ -199,7 +201,6 @@ protected virtual string GetLlamaccpArguments() if (remote) arguments += $" --port {port} --host 0.0.0.0"; if (numThreads > 0) arguments += $" -t {numThreads}"; if (loraPath != "") arguments += $" --lora \"{loraPath}\""; - arguments += $" --slot-save-path \"{slotSaveDir}\""; arguments += $" -ngl {numGPULayers}"; return arguments; } @@ -211,15 +212,15 @@ protected virtual string GetLlamaccpArguments() public async void Awake() { if (!enabled) return; - slotSaveDir = Application.persistentDataPath; if (asynchronousStartup) await Task.Run(() => StartLLMServer()); else StartLLMServer(); if (dontDestroyOnLoad) DontDestroyOnLoad(transform.root.gameObject); + if (basePrompt != "") await SetBasePrompt(basePrompt); } private void SetupLogging() { - logStreamWrapper = ConstructStreamWrapper(Debug.LogWarning, true); + logStreamWrapper = ConstructStreamWrapper(LLMUnitySetup.LogWarning, true); llmlib?.Logging(logStreamWrapper.GetStringWrapper()); } @@ -237,7 +238,7 @@ private void StartLLMServer() string arguments = GetLlamaccpArguments(); if (arguments == null) return; bool useGPU = numGPULayers > 0; - Debug.Log($"Server command: {arguments}"); + LLMUnitySetup.Log($"Server command: {arguments}"); foreach (string arch in LLMLib.PossibleArchitectures(useGPU)) { @@ -246,7 +247,7 @@ private void StartLLMServer() { InitLib(arch); InitServer(arguments); - Debug.Log($"Using architecture: {arch}"); + LLMUnitySetup.Log($"Using architecture: {arch}"); break; } catch (LLMException e) @@ -258,16 +259,16 @@ private void StartLLMServer() { error = $"{e.GetType()}: {e.Message}"; } - Debug.Log($"Tried architecture: {arch}, " + error); + LLMUnitySetup.Log($"Tried architecture: {arch}, " + error); } if (llmlib == null) { - Debug.LogError("LLM service couldn't be created"); + LLMUnitySetup.LogError("LLM service couldn't be created"); failed = true; return; } StartService(); - Debug.Log("LLM service created"); + LLMUnitySetup.Log("LLM service created"); } private void InitLib(string arch) @@ -343,7 +344,7 @@ void AssertStarted() else if (!started) error = "LLM service not started"; if (error != null) { - Debug.LogError(error); + LLMUnitySetup.LogError(error); throw new Exception(error); } } @@ -358,12 +359,12 @@ void CheckLLMStatus(bool log = true) string message = $"LLM {status}: {result}"; if (status > 0) { - if (log) Debug.LogError(message); + if (log) LLMUnitySetup.LogError(message); throw new LLMException(message, status); } else if (status < 0) { - if (log) Debug.LogWarning(message); + if (log) LLMUnitySetup.LogWarning(message); } } @@ -432,6 +433,7 @@ public async Task Slot(string json) public async Task Completion(string json, Callback streamCallback = null) { AssertStarted(); + if (streamCallback == null) streamCallback = (string s) => {}; StreamWrapper streamWrapper = ConstructStreamWrapper(streamCallback); await Task.Run(() => llmlib.LLM_Completion(LLMObject, json, streamWrapper.GetStringWrapper())); if (!started) return null; @@ -442,6 +444,13 @@ public async Task Completion(string json, Callback streamCallbac return result; } + public async Task SetBasePrompt(string base_prompt) + { + AssertStarted(); + SystemPromptRequest request = new SystemPromptRequest(){system_prompt = base_prompt, prompt = " ", n_predict = 0}; + await Completion(JsonUtility.ToJson(request)); + } + /// /// Allows to cancel the requests in a specific slot of the LLM /// @@ -479,7 +488,7 @@ public void Destroy() } catch (Exception e) { - Debug.LogError(e.Message); + LLMUnitySetup.LogError(e.Message); } } diff --git a/Runtime/LLMCharacter.cs b/Runtime/LLMCharacter.cs index 4c3400a3..8d36ead9 100644 --- a/Runtime/LLMCharacter.cs +++ b/Runtime/LLMCharacter.cs @@ -140,9 +140,10 @@ public void Awake() if (!enabled) return; if (!remote) { + AssignLLM(); if (llm == null) { - Debug.LogError("No llm provided!"); + LLMUnitySetup.LogError($"No LLM assigned or detected for LLMCharacter {name}!"); return; } id_slot = llm.Register(this); @@ -152,6 +153,54 @@ public void Awake() InitHistory(); } + void OnValidate() + { + AssignLLM(); + } + + void Reset() + { + AssignLLM(); + } + + void AssignLLM() + { + if (remote || llm != null) return; + + LLM[] existingLLMs = FindObjectsOfType(); + if (existingLLMs.Length == 0) return; + + SortBySceneAndHierarchy(existingLLMs); + llm = existingLLMs[0]; + string msg = $"Assigning LLM {llm.name} to LLMCharacter {name}"; + if (llm.gameObject.scene != gameObject.scene) msg += $" from scene {llm.gameObject.scene}"; + LLMUnitySetup.Log(msg); + } + + void SortBySceneAndHierarchy(LLM[] array) + { + for (int i = 0; i < array.Length - 1; i++) + { + bool swapped = false; + for (int j = 0; j < array.Length - i - 1; j++) + { + bool sameScene = array[j].gameObject.scene == array[j + 1].gameObject.scene; + bool swap = ( + (!sameScene && array[j + 1].gameObject.scene == gameObject.scene) || + (sameScene && array[j].transform.GetSiblingIndex() > array[j + 1].transform.GetSiblingIndex()) + ); + if (swap) + { + LLM temp = array[j]; + array[j] = array[j + 1]; + array[j + 1] = temp; + swapped = true; + } + } + if (!swapped) break; + } + } + protected void InitHistory() { InitPrompt(); @@ -174,7 +223,7 @@ protected async Task LoadHistory() string GetSavePath(string filename) { - return Path.Combine(Application.persistentDataPath, filename); + return Path.Combine(Application.persistentDataPath, filename).Replace('\\', '/'); } string GetJsonSavePath(string filename) @@ -182,10 +231,10 @@ string GetJsonSavePath(string filename) return GetSavePath(filename + ".json"); } - string GetCacheName(string filename) + string GetCacheSavePath(string filename) { // this is saved already in the Application.persistentDataPath folder - return filename + ".cache"; + return GetSavePath(filename + ".cache"); } private void InitPrompt(bool clearChat = true) @@ -290,7 +339,7 @@ ChatRequest GenerateRequest(string prompt) { // setup the request struct ChatRequest chatRequest = new ChatRequest(); - if (debugPrompt) Debug.Log(prompt); + if (debugPrompt) LLMUnitySetup.Log(prompt); chatRequest.prompt = prompt; chatRequest.id_slot = id_slot; chatRequest.temperature = temperature; @@ -321,18 +370,18 @@ ChatRequest GenerateRequest(string prompt) return chatRequest; } - private void AddMessage(string role, string content) + public void AddMessage(string role, string content) { // add the question / answer to the chat list, update prompt chat.Add(new ChatMessage { role = role, content = content }); } - private void AddPlayerMessage(string content) + public void AddPlayerMessage(string content) { AddMessage(playerName, content); } - private void AddAIMessage(string content) + public void AddAIMessage(string content) { AddMessage(AIName, content); } @@ -522,11 +571,11 @@ public async Task Detokenize(List tokens, Callback callback return await PostRequest(json, "detokenize", DetokenizeContent, callback); } - private async Task Slot(string filename, string action) + private async Task Slot(string filepath, string action) { SlotRequest slotRequest = new SlotRequest(); slotRequest.id_slot = id_slot; - slotRequest.filename = filename; + slotRequest.filepath = filepath; slotRequest.action = action; string json = JsonUtility.ToJson(slotRequest); return await PostRequest(json, "slots", SlotContent); @@ -540,12 +589,14 @@ private async Task Slot(string filename, string action) public async Task Save(string filename) { string filepath = GetJsonSavePath(filename); + string dirname = Path.GetDirectoryName(filepath); + if (!Directory.Exists(dirname)) Directory.CreateDirectory(dirname); string json = JsonUtility.ToJson(new ChatListWrapper { chat = chat }); File.WriteAllText(filepath, json); - string cachename = GetCacheName(filename); + string cachepath = GetCacheSavePath(filename); if (remote || !saveCache) return null; - string result = await Slot(cachename, "save"); + string result = await Slot(cachepath, "save"); return result; } @@ -559,16 +610,16 @@ public async Task Load(string filename) string filepath = GetJsonSavePath(filename); if (!File.Exists(filepath)) { - Debug.LogError($"File {filepath} does not exist."); + LLMUnitySetup.LogError($"File {filepath} does not exist."); return null; } string json = File.ReadAllText(filepath); chat = JsonUtility.FromJson(json).chat; - Debug.Log($"Loaded {filepath}"); + LLMUnitySetup.Log($"Loaded {filepath}"); - string cachename = GetCacheName(filename); - if (remote || !saveCache || !File.Exists(GetSavePath(cachename))) return null; - string result = await Slot(cachename, "restore"); + string cachepath = GetCacheSavePath(filename); + if (remote || !saveCache || !File.Exists(GetSavePath(cachepath))) return null; + string result = await Slot(cachepath, "restore"); return result; } @@ -646,14 +697,14 @@ protected async Task PostRequestLocal(string json, string endpoin } else { - Debug.LogError($"wrong callback type, should be string"); + LLMUnitySetup.LogError($"wrong callback type, should be string"); } callbackCalled = true; } callResult = await llm.Completion(json, callbackString); break; default: - Debug.LogError($"Unknown endpoint {endpoint}"); + LLMUnitySetup.LogError($"Unknown endpoint {endpoint}"); break; } @@ -668,7 +719,7 @@ protected async Task PostRequestRemote(string json, string endpoi // this function has streaming functionality i.e. handles the answer while it is being received if (endpoint == "slots") { - Debug.LogError("Saving and loading is not currently supported in remote setting"); + LLMUnitySetup.LogError("Saving and loading is not currently supported in remote setting"); return default; } @@ -702,7 +753,7 @@ protected async Task PostRequestRemote(string json, string endpoi await Task.Yield(); } WIPRequests.Remove(request); - if (request.result != UnityWebRequest.Result.Success) Debug.LogError(request.error); + if (request.result != UnityWebRequest.Result.Success) LLMUnitySetup.LogError(request.error); else result = ConvertContent(request.downloadHandler.text, getContent); callback?.Invoke(result); } diff --git a/Runtime/LLMChatTemplates.cs b/Runtime/LLMChatTemplates.cs index edad8d30..39f7fa9e 100644 --- a/Runtime/LLMChatTemplates.cs +++ b/Runtime/LLMChatTemplates.cs @@ -53,18 +53,18 @@ static ChatTemplate() chatTemplates = new Dictionary(); foreach (ChatTemplate template in templateClasses) { - if (templates.ContainsKey(template.GetName())) Debug.LogError($"{template.GetName()} already in templates"); + if (templates.ContainsKey(template.GetName())) LLMUnitySetup.LogError($"{template.GetName()} already in templates"); templates[template.GetName()] = template; - if (templatesDescription.ContainsKey(template.GetDescription())) Debug.LogError($"{template.GetDescription()} already in templatesDescription"); + if (templatesDescription.ContainsKey(template.GetDescription())) LLMUnitySetup.LogError($"{template.GetDescription()} already in templatesDescription"); templatesDescription[template.GetDescription()] = template.GetName(); foreach (string match in template.GetNameMatches()) { - if (modelTemplates.ContainsKey(match)) Debug.LogError($"{match} already in modelTemplates"); + if (modelTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"{match} already in modelTemplates"); modelTemplates[match] = template.GetName(); } foreach (string match in template.GetChatTemplateMatches()) { - if (chatTemplates.ContainsKey(match)) Debug.LogError($"{match} already in chatTemplates"); + if (chatTemplates.ContainsKey(match)) LLMUnitySetup.LogError($"{match} already in chatTemplates"); chatTemplates[match] = template.GetName(); } } @@ -125,7 +125,7 @@ public static string FromGGUF(string path) name = FromName(Path.GetFileNameWithoutExtension(path)); if (name != null) return name; - Debug.Log("No chat template could be matched, fallback to ChatML"); + LLMUnitySetup.Log("No chat template could be matched, fallback to ChatML"); return DefaultTemplate; } @@ -251,7 +251,7 @@ public override string[] GetStop(string playerName, string AIName) public class LLama2ChatTemplate : LLama2Template { public override string GetName() { return "llama chat"; } - public override string GetDescription() { return "llama 2 (modified for chat)"; } + public override string GetDescription() { return "llama 2 (chat)"; } public override string[] GetNameMatches() { return new string[] {"llama-2", "llama v2"}; } protected override string PlayerPrefix(string playerName) { return "### " + playerName + ":"; } @@ -275,7 +275,6 @@ public class LLama3ChatTemplate : ChatTemplate public override string[] GetNameMatches() { return new string[] {"llama-3", "llama v3"}; } public override string[] GetChatTemplateMatches() { return new string[] {"{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"};} - protected override string PromptPrefix() { return "<|begin_of_text|>"; } protected override string SystemPrefix() { return "<|start_header_id|>system<|end_header_id|>\n\n"; } protected override string SystemSuffix() { return "<|eot_id|>"; } @@ -300,7 +299,6 @@ public class MistralInstructTemplate : ChatTemplate public override string GetName() { return "mistral instruct"; } public override string GetDescription() { return "mistral instruct"; } - protected override string PromptPrefix() { return ""; } protected override string SystemPrefix() { return ""; } protected override string SystemSuffix() { return "\n\n"; } protected override string RequestPrefix() { return "[INST] "; } @@ -320,7 +318,7 @@ public override string[] GetStop(string playerName, string AIName) public class MistralChatTemplate : MistralInstructTemplate { public override string GetName() { return "mistral chat"; } - public override string GetDescription() { return "mistral (modified for chat)"; } + public override string GetDescription() { return "mistral (chat)"; } public override string[] GetNameMatches() { return new string[] {"mistral"}; } public override string[] GetChatTemplateMatches() { return new string[] {"{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"}; } @@ -413,7 +411,6 @@ public class Phi3Template : ChatTemplate public override string[] GetNameMatches() { return new string[] {"phi-3"}; } public override string[] GetChatTemplateMatches() { return new string[] {"{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}"}; } - protected override string PromptPrefix() { return ""; } protected override string PlayerPrefix(string playerName) { return $"<|user|>\n"; } protected override string AIPrefix(string AIName) { return $"<|assistant|>\n"; } protected override string RequestSuffix() { return "<|end|>\n"; } diff --git a/Runtime/LLMInterface.cs b/Runtime/LLMInterface.cs index 21ac8f36..f5086595 100644 --- a/Runtime/LLMInterface.cs +++ b/Runtime/LLMInterface.cs @@ -39,6 +39,14 @@ public struct ChatRequest public List messages; } + [Serializable] + public struct SystemPromptRequest + { + public string prompt; + public string system_prompt; + public int n_predict; + } + [Serializable] public struct ChatResult { @@ -96,7 +104,7 @@ public struct SlotRequest { public int id_slot; public string action; - public string filename; + public string filepath; } [Serializable] diff --git a/Runtime/LLMLib.cs b/Runtime/LLMLib.cs index 032b242f..45fc4260 100644 --- a/Runtime/LLMLib.cs +++ b/Runtime/LLMLib.cs @@ -246,7 +246,7 @@ static LLMLib() archCheckerHandle = LibraryLoader.LoadLibrary(archCheckerPath); if (archCheckerHandle == IntPtr.Zero) { - Debug.LogError($"Failed to load library {archCheckerPath}."); + LLMUnitySetup.LogError($"Failed to load library {archCheckerPath}."); } else { @@ -259,7 +259,7 @@ static LLMLib() public LLMLib(string arch) { - Debug.Log(GetArchitecturePath(arch)); + LLMUnitySetup.Log(GetArchitecturePath(arch)); libraryHandle = LibraryLoader.LoadLibrary(GetArchitecturePath(arch)); if (libraryHandle == IntPtr.Zero) { @@ -315,7 +315,7 @@ public static List PossibleArchitectures(bool gpu = false) } catch (Exception e) { - Debug.LogError($"{e.GetType()}: {e.Message}"); + LLMUnitySetup.LogError($"{e.GetType()}: {e.Message}"); } architectures.Add("noavx"); } @@ -324,20 +324,20 @@ public static List PossibleArchitectures(bool gpu = false) string arch = RuntimeInformation.ProcessArchitecture.ToString().ToLower(); if (arch.Contains("arm")) { - architectures.Add("arm64-no_acc"); architectures.Add("arm64-acc"); + architectures.Add("arm64-no_acc"); } else { - if (arch != "x86" && arch != "x64") Debug.LogWarning($"Unknown architecture of processor {arch}! Falling back to x86_64"); - architectures.Add("x64-no_acc"); + if (arch != "x86" && arch != "x64") LLMUnitySetup.LogWarning($"Unknown architecture of processor {arch}! Falling back to x86_64"); architectures.Add("x64-acc"); + architectures.Add("x64-no_acc"); } } else { string error = "Unknown OS"; - Debug.LogError(error); + LLMUnitySetup.LogError(error); throw new Exception(error); } return architectures; @@ -379,7 +379,7 @@ public static string GetArchitecturePath(string arch) else { string error = "Unknown OS"; - Debug.LogError(error); + LLMUnitySetup.LogError(error); throw new Exception(error); } return Path.Combine(LLMUnitySetup.libraryPath, filename); diff --git a/Runtime/LLMUnitySetup.cs b/Runtime/LLMUnitySetup.cs index 09adddb2..74022dbc 100644 --- a/Runtime/LLMUnitySetup.cs +++ b/Runtime/LLMUnitySetup.cs @@ -3,11 +3,11 @@ using UnityEditor; using System.IO; using UnityEngine; -using Debug = UnityEngine.Debug; using System.Threading.Tasks; using System.Net; using System; using System.IO.Compression; +using System.Collections.Generic; /// @defgroup llm LLM /// @defgroup template Chat Templates @@ -46,6 +46,8 @@ public class LocalAttribute : PropertyAttribute {} public class ModelAttribute : PropertyAttribute {} public class ModelAdvancedAttribute : PropertyAttribute {} public class ChatAttribute : PropertyAttribute {} + public class ChatAdvancedAttribute : PropertyAttribute {} + public class LLMUnityAttribute : PropertyAttribute {} public class NotImplementedException : Exception { @@ -66,7 +68,7 @@ public class LLMUnitySetup { // DON'T CHANGE! the version is autocompleted with a GitHub action /// LLM for Unity version - public static string Version = "v2.0.2"; + public static string Version = "v2.0.3"; /// LlamaLib version public static string LlamaLibVersion = "v1.1.5"; /// LlamaLib url @@ -83,7 +85,68 @@ public class LLMUnitySetup ("Phi 3 (small, great)", "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf?download=true"), }; + /// Add callback function to call for error logs + public static void AddErrorCallBack(Callback callback) + { + errorCallbacks.Add(callback); + } + + /// Remove callback function added for error logs + public static void RemoveErrorCallBack(Callback callback) + { + errorCallbacks.Remove(callback); + } + + /// Remove all callback function added for error logs + public static void ClearErrorCallBacks() + { + errorCallbacks.Clear(); + } + /// \cond HIDE + public enum DebugModeType + { + All, + Warning, + Error, + None + } + [LLMUnity] public static DebugModeType DebugMode = DebugModeType.All; + static List> errorCallbacks = new List>(); + + public static void Log(string message) + { + if ((int)DebugMode > (int)DebugModeType.All) return; + Debug.Log(message); + } + + public static void LogWarning(string message) + { + if ((int)DebugMode > (int)DebugModeType.Warning) return; + Debug.LogWarning(message); + } + + public static void LogError(string message) + { + if ((int)DebugMode > (int)DebugModeType.Error) return; + Debug.LogError(message); + foreach (Callback errorCallback in errorCallbacks) errorCallback(message); + } + + static string DebugModeKey = "DebugMode"; + static void LoadDebugMode() + { + DebugMode = (DebugModeType)PlayerPrefs.GetInt(DebugModeKey, (int)DebugModeType.All); + } + + public static void SetDebugMode(DebugModeType newDebugMode) + { + if (DebugMode == newDebugMode) return; + DebugMode = newDebugMode; + PlayerPrefs.SetInt(DebugModeKey, (int)DebugMode); + PlayerPrefs.Save(); + } + public static string GetAssetPath(string relPath = "") { // Path to store llm server binaries and models @@ -91,13 +154,22 @@ public static string GetAssetPath(string relPath = "") } #if UNITY_EDITOR - [HideInInspector] public static float libraryProgress = 1; - [InitializeOnLoadMethod] - private static async Task InitializeOnLoad() + static async Task InitializeOnLoad() { await DownloadLibrary(); + LoadDebugMode(); + } +#else + [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.BeforeSceneLoad)] + void InitializeOnLoad() + { + LoadDebugMode(); } +#endif + +#if UNITY_EDITOR + [HideInInspector] public static float libraryProgress = 1; public class DownloadStatus { @@ -123,11 +195,11 @@ public static async Task DownloadFile( // download a file to the specified path if (File.Exists(savePath) && !overwrite) { - Debug.Log($"File already exists at: {savePath}"); + Log($"File already exists at: {savePath}"); } else { - Debug.Log($"Downloading {fileUrl}..."); + Log($"Downloading {fileUrl}..."); string tmpPath = Path.Combine(Application.temporaryCachePath, Path.GetFileName(savePath)); WebClient client = new WebClient(); @@ -146,7 +218,7 @@ public static async Task DownloadFile( Directory.CreateDirectory(Path.GetDirectoryName(savePath)); File.Move(tmpPath, savePath); AssetDatabase.StopAssetEditing(); - Debug.Log($"Download complete!"); + Log($"Download complete!"); } progresscallback?.Invoke(1f); @@ -157,7 +229,7 @@ public static async Task AddAsset(string assetPath, string basePath) { if (!File.Exists(assetPath)) { - Debug.LogError($"{assetPath} does not exist!"); + LogError($"{assetPath} does not exist!"); return null; } // add an asset to the basePath directory if it is not already there and return the relative path @@ -168,7 +240,7 @@ public static async Task AddAsset(string assetPath, string basePath) { // if the asset is not in the assets dir copy it over fullPath = Path.Combine(basePathSlash, Path.GetFileName(assetPath)); - Debug.Log($"copying {assetPath} to {fullPath}"); + Log($"copying {assetPath} to {fullPath}"); AssetDatabase.StartAssetEditing(); await Task.Run(() => { @@ -180,7 +252,7 @@ await Task.Run(() => File.Copy(assetPath, fullPath); }); AssetDatabase.StopAssetEditing(); - Debug.Log("copying complete!"); + Log("copying complete!"); } return fullPath.Substring(basePathSlash.Length + 1); } diff --git a/Samples~/ChatBot/ChatBot.cs b/Samples~/ChatBot/ChatBot.cs index b64b276a..0aeea759 100644 --- a/Samples~/ChatBot/ChatBot.cs +++ b/Samples~/ChatBot/ChatBot.cs @@ -163,7 +163,7 @@ public void ExitGame() bool onValidateWarning = true; void OnValidate() { - if (onValidateWarning && llmCharacter.llm.model == "") + if (onValidateWarning && !llmCharacter.remote && llmCharacter.llm != null && llmCharacter.llm.model == "") { Debug.LogWarning($"Please select a model in the {llmCharacter.llm.gameObject.name} GameObject!"); onValidateWarning = false; diff --git a/Samples~/ChatBot/Scene.unity b/Samples~/ChatBot/Scene.unity index 2e65baa6..07764b61 100644 --- a/Samples~/ChatBot/Scene.unity +++ b/Samples~/ChatBot/Scene.unity @@ -657,17 +657,18 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: advancedOptions: 0 + remote: 0 + port: 13333 numThreads: -1 numGPULayers: 0 debug: 0 parallelPrompts: -1 + asynchronousStartup: 1 dontDestroyOnLoad: 1 - port: 13333 model: lora: contextSize: 0 batchSize: 512 - remote: 0 SelectedModel: 0 modelProgress: 1 modelCopyProgress: 1 @@ -1134,8 +1135,9 @@ MonoBehaviour: host: localhost port: 13333 save: - stream: 1 + saveCache: 0 debugPrompt: 0 + stream: 1 grammar: cachePrompt: 1 seed: 0 diff --git a/Samples~/KnowledgeBaseGame/KnowledgeBaseGame.cs b/Samples~/KnowledgeBaseGame/KnowledgeBaseGame.cs index 98bc16cd..eda4ea21 100644 --- a/Samples~/KnowledgeBaseGame/KnowledgeBaseGame.cs +++ b/Samples~/KnowledgeBaseGame/KnowledgeBaseGame.cs @@ -166,13 +166,19 @@ public void CancelRequests() AIReplyComplete(); } + public void ExitGame() + { + Debug.Log("Exit button clicked"); + Application.Quit(); + } + bool onValidateWarning = true; void OnValidate() { if (onValidateWarning) { if (embedding.SelectedOption == 0) Debug.LogWarning($"Please select a model in the {embedding.gameObject.name} GameObject!"); - if (llmCharacter.llm.model == "") Debug.LogWarning($"Please select a model in the {llmCharacter.llm.gameObject.name} GameObject!"); + if (!llmCharacter.remote && llmCharacter.llm != null && llmCharacter.llm.model == "") Debug.LogWarning($"Please select a model in the {llmCharacter.llm.gameObject.name} GameObject!"); onValidateWarning = false; } } diff --git a/Samples~/KnowledgeBaseGame/Scene.unity b/Samples~/KnowledgeBaseGame/Scene.unity index f3a5f351..5f01ed7a 100644 --- a/Samples~/KnowledgeBaseGame/Scene.unity +++ b/Samples~/KnowledgeBaseGame/Scene.unity @@ -285,7 +285,7 @@ MonoBehaviour: m_OnClick: m_PersistentCalls: m_Calls: - - m_Target: {fileID: 0} + - m_Target: {fileID: 489627714} m_TargetAssemblyTypeName: LLMUnitySamples.KnowledgeBaseGame, Assembly-CSharp m_MethodName: CancelRequests m_Mode: 1 @@ -1049,6 +1049,85 @@ MonoBehaviour: Answer3: {fileID: 1278063795} llmCharacter: {fileID: 1275496423} embedding: {fileID: 1772403371} +--- !u!1 &495835622 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 495835623} + - component: {fileID: 495835625} + - component: {fileID: 495835624} + m_Layer: 5 + m_Name: Text (Legacy) + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &495835623 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 495835622} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1468962045} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.5, y: 0.5} + m_AnchorMax: {x: 0.5, y: 0.5} + m_AnchoredPosition: {x: 0, y: 2} + m_SizeDelta: {x: 25, y: 25} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &495835624 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 495835622} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 14 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 10 + m_MaxSize: 40 + m_Alignment: 4 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: "\xD7" +--- !u!222 &495835625 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 495835622} + m_CullTransparentMesh: 1 --- !u!1 &504847327 GameObject: m_ObjectHideFlags: 0 @@ -2762,6 +2841,7 @@ RectTransform: m_Children: - {fileID: 1614562620} - {fileID: 964853999} + - {fileID: 39745287} - {fileID: 1821864874} - {fileID: 283994678} - {fileID: 1171575133} @@ -2775,7 +2855,7 @@ RectTransform: - {fileID: 828313582} - {fileID: 964460962} - {fileID: 1723006217} - - {fileID: 39745287} + - {fileID: 1468962045} m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} @@ -4203,8 +4283,9 @@ MonoBehaviour: host: localhost port: 13333 save: - stream: 1 + saveCache: 0 debugPrompt: 0 + stream: 1 grammar: cachePrompt: 1 seed: 0 @@ -5018,6 +5099,139 @@ CanvasRenderer: m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1388328534} m_CullTransparentMesh: 1 +--- !u!1 &1468962044 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1468962045} + - component: {fileID: 1468962048} + - component: {fileID: 1468962047} + - component: {fileID: 1468962046} + m_Layer: 5 + m_Name: ExitButton + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1468962045 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1468962044} + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1.5, y: 1.5, z: 1.5} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 495835623} + m_Father: {fileID: 898411689} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 1, y: 1} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 25, y: 25} + m_Pivot: {x: 1, y: 1} +--- !u!114 &1468962046 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1468962044} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4e29b1a8efbd4b44bb3f3716e73f07ff, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Navigation: + m_Mode: 3 + m_WrapAround: 0 + m_SelectOnUp: {fileID: 0} + m_SelectOnDown: {fileID: 0} + m_SelectOnLeft: {fileID: 0} + m_SelectOnRight: {fileID: 0} + m_Transition: 1 + m_Colors: + m_NormalColor: {r: 1, g: 1, b: 1, a: 1} + m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1} + m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608} + m_ColorMultiplier: 1 + m_FadeDuration: 0.1 + m_SpriteState: + m_HighlightedSprite: {fileID: 0} + m_PressedSprite: {fileID: 0} + m_SelectedSprite: {fileID: 0} + m_DisabledSprite: {fileID: 0} + m_AnimationTriggers: + m_NormalTrigger: Normal + m_HighlightedTrigger: Highlighted + m_PressedTrigger: Pressed + m_SelectedTrigger: Selected + m_DisabledTrigger: Disabled + m_Interactable: 1 + m_TargetGraphic: {fileID: 1468962047} + m_OnClick: + m_PersistentCalls: + m_Calls: + - m_Target: {fileID: 489627714} + m_TargetAssemblyTypeName: LLMUnitySamples.KnowledgeBaseGame, Assembly-CSharp + m_MethodName: ExitGame + m_Mode: 1 + m_Arguments: + m_ObjectArgument: {fileID: 0} + m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine + m_IntArgument: 0 + m_FloatArgument: 0 + m_StringArgument: + m_BoolArgument: 0 + m_CallState: 1 +--- !u!114 &1468962047 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1468962044} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.24722636, g: 0.24722636, b: 0.24722636, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10913, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 0 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!222 &1468962048 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1468962044} + m_CullTransparentMesh: 1 --- !u!1 &1535596276 GameObject: m_ObjectHideFlags: 0 @@ -7238,17 +7452,19 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: advancedOptions: 0 + remote: 0 + port: 13333 numThreads: -1 numGPULayers: 0 debug: 0 parallelPrompts: -1 + asynchronousStartup: 1 dontDestroyOnLoad: 1 - port: 13333 model: lora: contextSize: 0 batchSize: 512 - remote: 0 + basePrompt: SelectedModel: 0 modelProgress: 1 modelCopyProgress: 1 diff --git a/Samples~/MultipleCharacters/MultipleCharacters.cs b/Samples~/MultipleCharacters/MultipleCharacters.cs index 7b78a317..82067e2c 100644 --- a/Samples~/MultipleCharacters/MultipleCharacters.cs +++ b/Samples~/MultipleCharacters/MultipleCharacters.cs @@ -81,7 +81,7 @@ public void ExitGame() bool onValidateWarning = true; void OnValidate() { - if (onValidateWarning && llmCharacter1.llm.model == "") + if (onValidateWarning && !llmCharacter1.remote && llmCharacter1.llm != null && llmCharacter1.llm.model == "") { Debug.LogWarning($"Please select a model in the {llmCharacter1.llm.gameObject.name} GameObject!"); onValidateWarning = false; diff --git a/Samples~/MultipleCharacters/Scene.unity b/Samples~/MultipleCharacters/Scene.unity index 338d1e1a..999f08a1 100644 --- a/Samples~/MultipleCharacters/Scene.unity +++ b/Samples~/MultipleCharacters/Scene.unity @@ -596,8 +596,9 @@ MonoBehaviour: host: localhost port: 13333 save: - stream: 1 + saveCache: 0 debugPrompt: 0 + stream: 1 grammar: cachePrompt: 1 seed: 0 @@ -1513,17 +1514,18 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: advancedOptions: 0 + remote: 0 + port: 13333 numThreads: -1 numGPULayers: 0 debug: 0 parallelPrompts: -1 + asynchronousStartup: 1 dontDestroyOnLoad: 1 - port: 13333 model: lora: contextSize: 0 batchSize: 512 - remote: 0 SelectedModel: 0 modelProgress: 1 modelCopyProgress: 1 @@ -1964,8 +1966,9 @@ MonoBehaviour: host: localhost port: 13333 save: - stream: 1 + saveCache: 0 debugPrompt: 0 + stream: 1 grammar: cachePrompt: 1 seed: 0 diff --git a/Samples~/SimpleInteraction/Scene.unity b/Samples~/SimpleInteraction/Scene.unity index dab69082..9e41c779 100644 --- a/Samples~/SimpleInteraction/Scene.unity +++ b/Samples~/SimpleInteraction/Scene.unity @@ -484,8 +484,9 @@ MonoBehaviour: host: localhost port: 13333 save: - stream: 1 + saveCache: 0 debugPrompt: 0 + stream: 1 grammar: cachePrompt: 1 seed: 0 @@ -1036,17 +1037,18 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: advancedOptions: 0 + remote: 0 + port: 13333 numThreads: -1 numGPULayers: 0 debug: 0 parallelPrompts: -1 + asynchronousStartup: 1 dontDestroyOnLoad: 1 - port: 13333 model: lora: contextSize: 0 batchSize: 512 - remote: 0 SelectedModel: 0 modelProgress: 1 modelCopyProgress: 1 diff --git a/Samples~/SimpleInteraction/SimpleInteraction.cs b/Samples~/SimpleInteraction/SimpleInteraction.cs index e0469dec..9c088c3b 100644 --- a/Samples~/SimpleInteraction/SimpleInteraction.cs +++ b/Samples~/SimpleInteraction/SimpleInteraction.cs @@ -50,7 +50,7 @@ public void ExitGame() bool onValidateWarning = true; void OnValidate() { - if (onValidateWarning && llmCharacter.llm.model == "") + if (onValidateWarning && !llmCharacter.remote && llmCharacter.llm != null && llmCharacter.llm.model == "") { Debug.LogWarning($"Please select a model in the {llmCharacter.llm.gameObject.name} GameObject!"); onValidateWarning = false; diff --git a/Tests/Runtime/TestLLM.cs b/Tests/Runtime/TestLLM.cs index df79c472..60947fcc 100644 --- a/Tests/Runtime/TestLLM.cs +++ b/Tests/Runtime/TestLLM.cs @@ -34,9 +34,9 @@ public async Task Init() string fullModelPath = LLMUnitySetup.GetAssetPath(modelPath); _ = LLMUnitySetup.DownloadFile(modelUrl, fullModelPath, false, null, null, false); await llm.SetModel(fullModelPath); - Assert.AreEqual(llm.model, modelPath); llm.parallelPrompts = 1; llm.SetTemplate("alpaca"); + llm.asynchronousStartup = false; llmCharacter = gameObject.AddComponent(); llmCharacter.llm = llm; diff --git a/Tests/Runtime/TestLLMChatTemplates.cs b/Tests/Runtime/TestLLMChatTemplates.cs index faf597ff..b48a2bef 100644 --- a/Tests/Runtime/TestLLMChatTemplates.cs +++ b/Tests/Runtime/TestLLMChatTemplates.cs @@ -31,7 +31,7 @@ public void TestMistralInstruct() { Assert.AreEqual( new MistralInstructTemplate().ComputePrompt(messages, "assistant"), - "[INST] you are a bot\n\nHello, how are you? [/INST]I'm doing great. How can I help you today?[INST] I'd like to show off how chat templating works! [/INST]chat template is awesome[INST] do you think so? [/INST]" + "[INST] you are a bot\n\nHello, how are you? [/INST]I'm doing great. How can I help you today?[INST] I'd like to show off how chat templating works! [/INST]chat template is awesome[INST] do you think so? [/INST]" ); } @@ -40,7 +40,7 @@ public void TestMistralChat() { Assert.AreEqual( new MistralChatTemplate().ComputePrompt(messages, "assistant"), - "[INST] you are a bot\n\n### user: Hello, how are you? [/INST]### assistant: I'm doing great. How can I help you today?[INST] ### user: I'd like to show off how chat templating works! [/INST]### assistant: chat template is awesome[INST] ### user: do you think so? [/INST]### assistant:" + "[INST] you are a bot\n\n### user: Hello, how are you? [/INST]### assistant: I'm doing great. How can I help you today?[INST] ### user: I'd like to show off how chat templating works! [/INST]### assistant: chat template is awesome[INST] ### user: do you think so? [/INST]### assistant:" ); } @@ -67,7 +67,7 @@ public void TestLLama3Chat() { Assert.AreEqual( new LLama3ChatTemplate().ComputePrompt(messages, "assistant"), - "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nyou are a bot<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI'm doing great. How can I help you today?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nI'd like to show off how chat templating works!<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nchat template is awesome<|eot_id|><|start_header_id|>user<|end_header_id|>\n\ndo you think so?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" + "<|start_header_id|>system<|end_header_id|>\n\nyou are a bot<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI'm doing great. How can I help you today?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nI'd like to show off how chat templating works!<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nchat template is awesome<|eot_id|><|start_header_id|>user<|end_header_id|>\n\ndo you think so?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" ); } @@ -103,7 +103,7 @@ public void TestPhi3() { Assert.AreEqual( new Phi3Template().ComputePrompt(messages, "assistant"), - "<|user|>\nyou are a bot\n\nHello, how are you?<|end|>\n<|assistant|>\nI'm doing great. How can I help you today?<|end|>\n<|user|>\nI'd like to show off how chat templating works!<|end|>\n<|assistant|>\nchat template is awesome<|end|>\n<|user|>\ndo you think so?<|end|>\n<|assistant|>\n" + "<|user|>\nyou are a bot\n\nHello, how are you?<|end|>\n<|assistant|>\nI'm doing great. How can I help you today?<|end|>\n<|user|>\nI'd like to show off how chat templating works!<|end|>\n<|assistant|>\nchat template is awesome<|end|>\n<|user|>\ndo you think so?<|end|>\n<|assistant|>\n" ); } diff --git a/VERSION b/VERSION index f3b15f3f..f256be60 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.2 +v2.0.3 diff --git a/package.json b/package.json index d144c411..bb83f63a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ai.undream.llm", - "version": "2.0.2", + "version": "2.0.3", "displayName": "LLM for Unity", "description": "LLM for Unity allows to run and distribute Large Language Models (LLMs) in the Unity engine.", "unity": "2022.3", @@ -37,7 +37,7 @@ }, { "displayName": "KnowledgeBaseGame", - "description": "Simple detective game using a knowledge base to provide information to the LLM.", + "description": "Simple detective game using a knowledge base to provide information to the LLM.\n**Requires installation of the RAGSearchUnity package (from UndreamAI)**", "path": "Samples~/KnowledgeBaseGame" }, {