From 952ad1edd5d4ec41bbb2dab53eee52654a3c5268 Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Wed, 21 Aug 2024 17:49:09 +0300 Subject: [PATCH 1/6] use CallWithLock function to lock lib calls --- Runtime/LLM.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Runtime/LLM.cs b/Runtime/LLM.cs index 9e50c513..140bc75f 100644 --- a/Runtime/LLM.cs +++ b/Runtime/LLM.cs @@ -354,7 +354,7 @@ private void StartLLMServer(string arguments) failed = true; return; } - CallIfNotDestroyed(() => StartService()); + CallWithLock(StartService); LLMUnitySetup.Log("LLM service created"); } @@ -364,22 +364,22 @@ private void InitLib(string arch) CheckLLMStatus(false); } - void CallIfNotDestroyed(EmptyCallback fn) + void CallWithLock(EmptyCallback fn, bool checkNull = true) { lock (startLock) { - if (llmlib == null) throw new DestroyException(); + if (checkNull && llmlib == null) throw new DestroyException(); fn(); } } private void InitService(string arguments) { - if (debug) CallIfNotDestroyed(() => SetupLogging()); - CallIfNotDestroyed(() => { LLMObject = llmlib.LLM_Construct(arguments); }); - if (remote) CallIfNotDestroyed(() => llmlib.LLM_StartServer(LLMObject)); - CallIfNotDestroyed(() => llmlib.LLM_SetTemplate(LLMObject, chatTemplate)); - CallIfNotDestroyed(() => CheckLLMStatus(false)); + if (debug) CallWithLock(SetupLogging); + CallWithLock(() => { LLMObject = llmlib.LLM_Construct(arguments); }); + if (remote) CallWithLock(() => llmlib.LLM_StartServer(LLMObject)); + CallWithLock(() => llmlib.LLM_SetTemplate(LLMObject, chatTemplate)); + CallWithLock(() => CheckLLMStatus(false)); } private void StartService() @@ -644,7 +644,7 @@ public void CancelRequest(int id_slot) /// public void Destroy() { - lock (startLock) + CallWithLock(() => { try { @@ -669,7 +669,7 @@ public void Destroy() { LLMUnitySetup.LogError(e.Message); } - } + }, false); } /// From db08fc4a84aea82f83b8da0c6fa28a04e7a0fbe6 Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Thu, 22 Aug 2024 12:34:52 +0300 Subject: [PATCH 2/6] remote request with retries --- Runtime/LLMCharacter.cs | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/Runtime/LLMCharacter.cs b/Runtime/LLMCharacter.cs index 3b002564..5732b1cb 100644 --- a/Runtime/LLMCharacter.cs +++ b/Runtime/LLMCharacter.cs @@ -27,6 +27,8 @@ public class LLMCharacter : MonoBehaviour [Remote] public string host = "localhost"; /// port to use for the LLM server [Remote] public int port = 13333; + /// number of retries to use for the LLM server requests (-1 = infinite) + [Remote] public int numRetries = -1; /// file to save the chat history. /// The file is saved only for Chat calls with addToHistory set to true. /// The file will be saved within the persistentDataPath directory (see https://docs.unity3d.com/ScriptReference/Application-persistentDataPath.html). @@ -750,38 +752,57 @@ protected async Task PostRequestRemote(string json, string endpoi Ret result = default; byte[] jsonToSend = new System.Text.UTF8Encoding().GetBytes(json); - using (var request = UnityWebRequest.Put($"{host}:{port}/{endpoint}", jsonToSend)) - { - WIPRequests.Add(request); + UnityWebRequest request = null; + string error = null; + int tryNr = numRetries; - request.method = "POST"; - if (requestHeaders != null) + while (tryNr != 0) + { + using (request = UnityWebRequest.Put($"{host}:{port}/{endpoint}", jsonToSend)) { - for (int i = 0; i < requestHeaders.Count; i++) - request.SetRequestHeader(requestHeaders[i].Item1, requestHeaders[i].Item2); - } + WIPRequests.Add(request); - // Start the request asynchronously - var asyncOperation = request.SendWebRequest(); - float lastProgress = 0f; - // Continue updating progress until the request is completed - while (!asyncOperation.isDone) - { - float currentProgress = request.downloadProgress; - // Check if progress has changed - if (currentProgress != lastProgress && callback != null) + request.method = "POST"; + if (requestHeaders != null) + { + for (int i = 0; i < requestHeaders.Count; i++) + request.SetRequestHeader(requestHeaders[i].Item1, requestHeaders[i].Item2); + } + + // Start the request asynchronously + var asyncOperation = request.SendWebRequest(); + float lastProgress = 0f; + // Continue updating progress until the request is completed + while (!asyncOperation.isDone) { - callback?.Invoke(ConvertContent(request.downloadHandler.text, getContent)); - lastProgress = currentProgress; + float currentProgress = request.downloadProgress; + // Check if progress has changed + if (currentProgress != lastProgress && callback != null) + { + callback?.Invoke(ConvertContent(request.downloadHandler.text, getContent)); + lastProgress = currentProgress; + } + // Wait for the next frame + await Task.Yield(); + } + WIPRequests.Remove(request); + if (request.result == UnityWebRequest.Result.Success) + { + result = ConvertContent(request.downloadHandler.text, getContent); + error = null; + break; + } + else + { + result = default; + error = request.error; } - // Wait for the next frame - await Task.Yield(); } - WIPRequests.Remove(request); - if (request.result != UnityWebRequest.Result.Success) LLMUnitySetup.LogError(request.error); - else result = ConvertContent(request.downloadHandler.text, getContent); - callback?.Invoke(result); + tryNr--; } + + if (error != null) LLMUnitySetup.LogError(error); + callback?.Invoke(result); return result; } From 8de7313e1f6ab7086eb0acdc549c9a1a5655d63c Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Thu, 22 Aug 2024 12:35:31 +0300 Subject: [PATCH 3/6] check template if null before using it --- Runtime/LLMCharacter.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Runtime/LLMCharacter.cs b/Runtime/LLMCharacter.cs index 5732b1cb..1bf68236 100644 --- a/Runtime/LLMCharacter.cs +++ b/Runtime/LLMCharacter.cs @@ -120,7 +120,7 @@ public class LLMCharacter : MonoBehaviour public List chat; private SemaphoreSlim chatLock = new SemaphoreSlim(1, 1); private string chatTemplate; - private ChatTemplate template; + private ChatTemplate template = null; public string grammarString; protected int id_slot = -1; private List<(string, string)> requestHeaders = new List<(string, string)> { ("Content-Type", "application/json") }; @@ -272,10 +272,21 @@ public void SetPrompt(string newPrompt, bool clearChat = true) InitPrompt(clearChat); } + private bool CheckTemplate() + { + if (template == null) + { + LLMUnitySetup.LogError("Template not set!"); + return false; + } + return true; + } + private async Task InitNKeep() { if (setNKeepToPrompt && nKeep == -1) { + if (!CheckTemplate()) return; string systemPrompt = template.ComputePrompt(new List(){chat[0]}, playerName, "", false); await Tokenize(systemPrompt, SetNKeep); } @@ -313,7 +324,7 @@ public async Task LoadTemplate() if (llmTemplate != chatTemplate) { chatTemplate = llmTemplate; - template = ChatTemplate.GetTemplate(chatTemplate); + template = chatTemplate == null ? null : ChatTemplate.GetTemplate(chatTemplate); } } @@ -333,6 +344,7 @@ public async void SetGrammar(string path) List GetStopwords() { + if (!CheckTemplate()) return null; List stopAll = new List(template.GetStop(playerName, AIName)); if (stop != null) stopAll.AddRange(stop); return stopAll; @@ -467,6 +479,7 @@ public async Task Chat(string query, Callback callback = null, E // call the callback function while the answer is received // call the completionCallback function when the answer is fully received await LoadTemplate(); + if (!CheckTemplate()) return null; await InitNKeep(); string json; From 23f8fe4d356c2087d8339c66e31f18d64446c1cc Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Thu, 22 Aug 2024 12:40:50 +0300 Subject: [PATCH 4/6] recompute nkeep when template changes --- Runtime/LLMCharacter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Runtime/LLMCharacter.cs b/Runtime/LLMCharacter.cs index 1bf68236..c5ce99fd 100644 --- a/Runtime/LLMCharacter.cs +++ b/Runtime/LLMCharacter.cs @@ -325,6 +325,7 @@ public async Task LoadTemplate() { chatTemplate = llmTemplate; template = chatTemplate == null ? null : ChatTemplate.GetTemplate(chatTemplate); + nKeep = -1; } } From 97a5b27e86b978c12d8134daf123a3c7df03d5e2 Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Thu, 22 Aug 2024 12:42:38 +0300 Subject: [PATCH 5/6] add retries to readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bdf6350b..8dee1a57 100644 --- a/README.md +++ b/README.md @@ -406,8 +406,9 @@ If the user's GPU is not supported, the LLM will fall back to the CPU - `Remote` whether the LLM used is remote or local - `LLM` the LLM GameObject (if `Remote` is not set) -- `Hort` ip of the LLM (if `Remote` is set) -- `Port` port of the LLM (if `Remote` is set) +- `Hort` ip of the LLM server (if `Remote` is set) +- `Port` port of the LLM server (if `Remote` is set) +- `Num Retries` number of HTTP request retries from the LLM server (if `Remote` is set) -
Save save filename or relative path If set, the chat history and LLM state (if save cache is enabled) is automatically saved to file specified.
The chat history is saved with a json suffix and the LLM state with a cache suffix.
Both files are saved in the [persistentDataPath folder of Unity](https://docs.unity3d.com/ScriptReference/Application-persistentDataPath.html).
- `Save Cache` select to save the LLM state along with the chat history. The LLM state is typically around 100MB+. - `Debug Prompt` select to log the constructed prompts in the Unity Editor From 573da20ea13e9dd6f8aabde45bd32630b211825c Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Thu, 22 Aug 2024 12:46:15 +0300 Subject: [PATCH 6/6] set template before starting server --- Runtime/LLM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Runtime/LLM.cs b/Runtime/LLM.cs index 140bc75f..77b71398 100644 --- a/Runtime/LLM.cs +++ b/Runtime/LLM.cs @@ -377,8 +377,8 @@ private void InitService(string arguments) { if (debug) CallWithLock(SetupLogging); CallWithLock(() => { LLMObject = llmlib.LLM_Construct(arguments); }); - if (remote) CallWithLock(() => llmlib.LLM_StartServer(LLMObject)); CallWithLock(() => llmlib.LLM_SetTemplate(LLMObject, chatTemplate)); + if (remote) CallWithLock(() => llmlib.LLM_StartServer(LLMObject)); CallWithLock(() => CheckLLMStatus(false)); }