improve build process

Author: amakropoulos

Editor/LLMBuildProcessor.cs

@@ -2,150 +2,58 @@
 using UnityEditor.Build;
 using UnityEditor.Build.Reporting;
 using UnityEngine;
-using System.IO;
-using System.Collections.Generic;
-using System;
 
 namespace LLMUnity
 {
-    public class LLMBuildProcessor : MonoBehaviour, IPreprocessBuildWithReport, IPostprocessBuildWithReport
+    public class LLMBuildProcessor : IPreprocessBuildWithReport, IPostprocessBuildWithReport
     {
         public int callbackOrder => 0;
-        static string tempDir = Path.Combine(Application.temporaryCachePath, "LLMBuildProcessor", Path.GetFileName(LLMUnitySetup.libraryPath));
-        static List<MovedPair> movedPairs = new List<MovedPair>();
-        static string movedCache = Path.Combine(tempDir, "moved.json");
 
-        [InitializeOnLoadMethod]
-        private static void InitializeOnLoad()
-        {
-            if (!Directory.Exists(tempDir)) Directory.CreateDirectory(tempDir);
-            else ResetMoves();
-        }
-
-        // CALLED BEFORE THE BUILD
+        // called before the build
         public void OnPreprocessBuild(BuildReport report)
         {
-            // Start listening for errors when build starts
             Application.logMessageReceived += OnBuildError;
-            HideLibraryPlatforms(report.summary.platform);
-            HideModels();
-            if (movedPairs.Count > 0) AssetDatabase.Refresh();
-        }
-
-        // CALLED DURING BUILD TO CHECK FOR ERRORS
-        private void OnBuildError(string condition, string stacktrace, LogType type)
-        {
-            if (type == LogType.Error)
-            {
-                // FAILED TO BUILD, STOP LISTENING FOR ERRORS
-                BuildCompleted();
-            }
-        }
-
-        // CALLED AFTER THE BUILD
-        public void OnPostprocessBuild(BuildReport report)
-        {
-            BuildCompleted();
-        }
-
-        public void BuildCompleted()
-        {
-            Application.logMessageReceived -= OnBuildError;
-            ResetMoves();
-        }
-
-        static bool MovePath(string source, string target)
-        {
-            bool moved = false;
-            if (File.Exists(source))
-            {
-                File.Move(source, target);
-                moved = true;
-            }
-            else if (Directory.Exists(source))
-            {
-                Directory.Move(source, target);
-                moved = true;
-            }
-            if (moved)
-            {
-                movedPairs.Add(new MovedPair {source = source, target = target});
-                File.WriteAllText(movedCache, JsonUtility.ToJson(new FoldersMovedWrapper { movedPairs = movedPairs }));
-            }
-            return moved;
-        }
-
-        static void MoveAssetAndMeta(string source, string target)
-        {
-            MovePath(source + ".meta", target + ".meta");
-            MovePath(source, target);
-        }
-
-        static void HideLibraryPlatforms(BuildTarget buildPlatform)
-        {
-            List<string> platforms = new List<string>(){ "windows", "macos", "linux", "android" };
-            switch (buildPlatform)
+            string platform = null;
+            switch (report.summary.platform)
             {
                 case BuildTarget.StandaloneWindows:
                 case BuildTarget.StandaloneWindows64:
-                    platforms.Remove("windows");
+                    platform = "windows";
                     break;
                 case BuildTarget.StandaloneLinux64:
-                    platforms.Remove("linux");
+                    platform = "linux";
                     break;
                 case BuildTarget.StandaloneOSX:
-                    platforms.Remove("macos");
+                    platform = "macos";
                     break;
                 case BuildTarget.Android:
-                    platforms.Remove("android");
+                    platform = "android";
+                    break;
+                case BuildTarget.iOS:
+                    platform = "ios";
                     break;
             }
-
-            foreach (string dirname in Directory.GetDirectories(LLMUnitySetup.libraryPath))
-            {
-                foreach (string platform in platforms)
-                {
-                    if (Path.GetFileName(dirname).StartsWith(platform))
-                    {
-                        MoveAssetAndMeta(dirname, Path.Combine(tempDir, Path.GetFileName(dirname)));
-                    }
-                }
-            }
+            LLMBuilder.HideLibraryPlatforms(platform);
+            LLMBuilder.CopyModels();
+            AssetDatabase.Refresh();
         }
 
-        static void HideModels()
+        // called during build to check for errors
+        private void OnBuildError(string condition, string stacktrace, LogType type)
         {
-            foreach (LLM llm in FindObjectsOfType<LLM>())
-            {
-                // if (!llm.downloadOnBuild) continue;
-                // if (llm.modelURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.model), Path.Combine(tempDir, Path.GetFileName(llm.model)));
-                if (llm.loraURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.lora), Path.Combine(tempDir, Path.GetFileName(llm.lora)));
-            }
+            if (type == LogType.Error) BuildCompleted();
         }
 
-        static void ResetMoves()
+        // called after the build
+        public void OnPostprocessBuild(BuildReport report)
         {
-            if (!File.Exists(movedCache)) return;
-            List<MovedPair> movedPairs = JsonUtility.FromJson<FoldersMovedWrapper>(File.ReadAllText(movedCache)).movedPairs;
-            if (movedPairs == null) return;
-
-            bool refresh = false;
-            foreach (var pair in movedPairs) refresh |= MovePath(pair.target, pair.source);
-            if (refresh) AssetDatabase.Refresh();
-            File.Delete(movedCache);
+            BuildCompleted();
         }
-    }
 
-    [Serializable]
-    public struct MovedPair
-    {
-        public string source;
-        public string target;
-    }
-
-    [Serializable]
-    public class FoldersMovedWrapper
-    {
-        public List<MovedPair> movedPairs;
+        public void BuildCompleted()
+        {
+            Application.logMessageReceived -= OnBuildError;
+            LLMBuilder.Reset();
+        }
     }
 }

Editor/LLMEditor.cs

@@ -63,7 +63,7 @@ public void AddModelLoaders(SerializedObject llmScriptSO, LLM llmScript)
                 }
                 EditorGUILayout.EndHorizontal();
             }
-            AddLoadButtons();
+            _ = AddLoadButtons();
             bool downloadOnStart = EditorGUILayout.Toggle("Download on Start", LLMManager.downloadOnStart);
             if (downloadOnStart != LLMManager.downloadOnStart)
             {

Runtime/LLMUnitySetup.cs

@@ -78,10 +78,14 @@ public class LLMUnitySetup
         public static string LlamaLibURL = $"https://github.com/undreamai/LlamaLib/releases/download/{LlamaLibVersion}/undreamai-{LlamaLibVersion}-llamacpp.zip";
         /// <summary> LlamaLib path </summary>
         public static string libraryPath = GetAssetPath(Path.GetFileName(LlamaLibURL).Replace(".zip", ""));
+        /// <summary> LLMnity store path </summary>
+        public static string LLMUnityStore = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity");
         /// <summary> Model download path </summary>
-        public static string modelDownloadPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity");
+        public static string modelDownloadPath = Path.Combine(LLMUnityStore, "models");
         /// <summary> Model list for project </summary>
         public static string modelListPath = Path.Combine(Application.temporaryCachePath, "modelCache.json");
+        /// <summary> Temporary dir for build </summary>
+        public static string buildTempDir = Path.Combine(Application.temporaryCachePath, "LLMUnityBuild");
 
         /// <summary> Default models for download </summary>
         [HideInInspector] public static readonly (string, string)[] modelOptions = new(string, string)[]
@@ -228,11 +232,11 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f
             string target = GetAssetPath(assetName);
             if (!overwrite && File.Exists(target))
             {
-                Debug.Log($"File {target} already exists");
+                Log($"File {target} already exists");
                 return;
             }
 
-            Debug.Log($"Extracting {source} to {target}");
+            Log($"Extracting {source} to {target}");
 
             // UnityWebRequest to read the file from StreamingAssets
             UnityWebRequest www = UnityWebRequest.Get(source);
@@ -242,7 +246,7 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f
             while (!operation.isDone) await Task.Delay(1);
             if (www.result != UnityWebRequest.Result.Success)
             {
-                Debug.LogError("Failed to load file from StreamingAssets: " + www.error);
+                LogError("Failed to load file from StreamingAssets: " + www.error);
             }
             else
             {
@@ -267,6 +271,44 @@ public static bool IsSubPath(string childPath, string parentPath)
         }
 
 #if UNITY_EDITOR
+
+        public static void CopyPath(string source, string target)
+        {
+            if (File.Exists(source))
+            {
+                File.Copy(source, target);
+            }
+            else if (Directory.Exists(source))
+            {
+                Directory.CreateDirectory(target);
+                List<string> filesAndDirs = new List<string>();
+                filesAndDirs.AddRange(Directory.GetFiles(source));
+                filesAndDirs.AddRange(Directory.GetDirectories(source));
+                foreach (string path in filesAndDirs)
+                {
+                    CopyPath(path, Path.Combine(target, Path.GetFileName(path)));
+                }
+            }
+        }
+
+        public static void MovePath(string source, string target)
+        {
+            CopyPath(source, target);
+            DeletePath(source);
+        }
+
+        public static bool DeletePath(string path)
+        {
+            if (!IsSubPath(path, GetAssetPath()) && !IsSubPath(path, buildTempDir))
+            {
+                LogError($"Safeguard: {path} will not be deleted because it may not be safe");
+                return false;
+            }
+            if (File.Exists(path)) File.Delete(path);
+            else if (Directory.Exists(path)) Directory.Delete(path, true);
+            return true;
+        }
+
         [HideInInspector] public static float libraryProgress = 1;
 
         private static async Task DownloadLibrary()
@@ -277,7 +319,9 @@ private static async Task DownloadLibrary()
             if (!Directory.Exists(libraryPath))
             {
                 await DownloadFile(LlamaLibURL, libZip, true, null, SetLibraryProgress);
+                AssetDatabase.StartAssetEditing();
                 ZipFile.ExtractToDirectory(libZip, libraryPath);
+                AssetDatabase.StopAssetEditing();
                 File.Delete(libZip);
             }
             libraryProgress = 1;
@@ -307,26 +351,6 @@ public static string AddAsset(string assetPath)
             return filename;
         }
 
-        public static void CreateSymlink(string sourcePath, string targetPath)
-        {
-            bool isDirectory = Directory.Exists(sourcePath);
-            if (!isDirectory && !File.Exists(sourcePath)) throw new FileNotFoundException($"Source path does not exist: {sourcePath}");
-
-            bool success;
-#if UNITY_STANDALONE_WIN
-            success = CreateSymbolicLink(targetPath, sourcePath, (int)isDirectory);
-#else
-            success = symlink(sourcePath, targetPath) == 0;
-#endif
-            if (!success) throw new IOException($"Failed to create symbolic link: {targetPath}");
-        }
-
-        [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
-        private static extern bool CreateSymbolicLink(string lpSymlinkFileName, string lpTargetFileName, int dwFlags);
-
-        [DllImport("libc", SetLastError = true)]
-        private static extern int symlink(string oldpath, string newpath);
-
 #endif
         /// \endcond
         public static int GetMaxFreqKHz(int cpuId)
@@ -422,7 +446,7 @@ public static int AndroidGetNumBigCores()
             }
             catch (Exception e)
             {
-                Debug.LogError(e.Message);
+                LogError(e.Message);
             }
 
             int numBigCores = 0;
@@ -474,7 +498,7 @@ public static int AndroidGetNumBigCoresCapacity()
             }
             catch (Exception e)
             {
-                Debug.LogError(e.Message);
+                LogError(e.Message);
             }
 
             int numBigCores = 0;