Skip to content

Commit fdc53bf

Browse files
amakropoulosamakropoulos
committed
WIP
1 parent 891497b commit fdc53bf

7 files changed

Lines changed: 153 additions & 151 deletions

File tree

‎Editor/LLMEditor.cs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public override void AddModelSettings(SerializedObject llmScriptSO)
111111
if (llmScriptSO.FindProperty("advancedOptions").boolValue)
112112
{
113113
attributeClasses.Add(typeof(ModelAdvancedAttribute));
114-
if (LLMUnitySetup.FullLlamaLib) attributeClasses.Add(typeof(ModelExtrasAttribute));
114+
attributeClasses.Add(typeof(ModelExtrasAttribute));
115115
}
116116
ShowPropertiesOfClass("", llmScriptSO, attributeClasses, false);
117117
Space();
@@ -126,10 +126,10 @@ static void ResetModelOptions()
126126
{
127127
List<string> existingOptions = new List<string>();
128128
foreach (ModelEntry entry in LLMManager.modelEntries) existingOptions.Add(entry.url);
129-
modelOptions = new List<string>(){"Download model", "Custom URL"};
130-
modelNames = new List<string>(){null, null};
131-
modelURLs = new List<string>(){null, null};
132-
modelLicenses = new List<string>(){null, null};
129+
modelOptions = new List<string>() { "Download model", "Custom URL" };
130+
modelNames = new List<string>() { null, null };
131+
modelURLs = new List<string>() { null, null };
132+
modelLicenses = new List<string>() { null, null };
133133
foreach (var entry in LLMUnitySetup.modelOptions)
134134
{
135135
string category = entry.Key;
@@ -146,9 +146,9 @@ static void ResetModelOptions()
146146

147147
float[] GetColumnWidths(bool expandedView)
148148
{
149-
List<float> widths = new List<float>(){actionColumnWidth, nameColumnWidth, templateColumnWidth};
150-
if (expandedView) widths.AddRange(new List<float>(){textColumnWidth, textColumnWidth});
151-
widths.AddRange(new List<float>(){includeInBuildColumnWidth, actionColumnWidth});
149+
List<float> widths = new List<float>() { actionColumnWidth, nameColumnWidth, templateColumnWidth };
150+
if (expandedView) widths.AddRange(new List<float>() { textColumnWidth, textColumnWidth });
151+
widths.AddRange(new List<float>() { includeInBuildColumnWidth, actionColumnWidth });
152152
return widths.ToArray();
153153
}
154154

@@ -420,7 +420,7 @@ void OnEnable()
420420
EditorGUI.LabelField(rects[col++], "Build");
421421
EditorGUI.LabelField(rects[col++], "");
422422
},
423-
drawFooterCallback = {},
423+
drawFooterCallback = { },
424424
footerHeight = 0,
425425
};
426426
}
@@ -440,14 +440,14 @@ private void DrawCopyableLabel(Rect rect, string label, string text = "")
440440

441441
private void CopyToClipboard(string text)
442442
{
443-
TextEditor te = new TextEditor {text = text};
443+
TextEditor te = new TextEditor { text = text };
444444
te.SelectAll();
445445
te.Copy();
446446
}
447447

448448
public void AddExtrasToggle()
449449
{
450-
if (ToggleButton("Use extras", LLMUnitySetup.FullLlamaLib)) LLMUnitySetup.SetFullLlamaLib(!LLMUnitySetup.FullLlamaLib);
450+
if (ToggleButton("Use cuBLAS", LLMUnitySetup.CUBLAS)) LLMUnitySetup.SetCUBLAS(!LLMUnitySetup.CUBLAS);
451451
}
452452

453453
public override void AddOptionsToggles(SerializedObject llmScriptSO)

‎Runtime/LLM.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ protected virtual string GetLlamaccpArguments()
445445
if (numThreadsToUse > 0) arguments += $" -t {numThreadsToUse}";
446446
arguments += loraArgument;
447447
if (numGPULayers > 0) arguments += $" -ngl {numGPULayers}";
448-
if (LLMUnitySetup.FullLlamaLib && flashAttention) arguments += $" --flash-attn";
448+
if (flashAttention) arguments += $" --flash-attn";
449449
if (remote)
450450
{
451451
arguments += $" --port {port} --host 0.0.0.0";
@@ -556,7 +556,7 @@ private void StartService()
556556
{
557557
llmThread = new Thread(() => llmlib.LLM_Start(LLMObject));
558558
llmThread.Start();
559-
while (!llmlib.LLM_Started(LLMObject)) {}
559+
while (!llmlib.LLM_Started(LLMObject)) { }
560560
ApplyLoras();
561561
started = true;
562562
}
@@ -780,7 +780,7 @@ public async Task<string> Slot(string json)
780780
public async Task<string> Completion(string json, Callback<string> streamCallback = null)
781781
{
782782
AssertStarted();
783-
if (streamCallback == null) streamCallback = (string s) => {};
783+
if (streamCallback == null) streamCallback = (string s) => { };
784784
StreamWrapper streamWrapper = ConstructStreamWrapper(streamCallback);
785785
await Task.Run(() => llmlib.LLM_Completion(LLMObject, json, streamWrapper.GetStringWrapper()));
786786
if (!started) return null;

‎Runtime/LLMBuilder.cs‎

Lines changed: 60 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public static void MovePath(string source, string target)
9898
/// <param name="path">path</param>
9999
public static bool DeletePath(string path)
100100
{
101-
string[] allowedDirs = new string[] { LLMUnitySetup.GetAssetPath(), BuildTempDir, PluginDir("Android"), PluginDir("iOS"), PluginDir("VisionOS")};
101+
string[] allowedDirs = new string[] { LLMUnitySetup.GetAssetPath(), BuildTempDir, PluginDir("Android"), PluginDir("iOS"), PluginDir("VisionOS") };
102102
bool deleteOK = false;
103103
foreach (string allowedDir in allowedDirs) deleteOK = deleteOK || LLMUnitySetup.IsSubPath(path, allowedDir);
104104
if (!deleteOK)
@@ -113,7 +113,7 @@ public static bool DeletePath(string path)
113113

114114
static void AddMovedPair(string source, string target)
115115
{
116-
movedPairs.Add(new StringPair {source = source, target = target});
116+
movedPairs.Add(new StringPair { source = source, target = target });
117117
File.WriteAllText(movedCache, JsonUtility.ToJson(new ListStringPair { pairs = movedPairs }, true));
118118
}
119119

@@ -164,70 +164,102 @@ static void AddActionAddMeta(string target)
164164
/// <param name="platform">target platform</param>
165165
public static void BuildLibraryPlatforms(BuildTarget buildTarget)
166166
{
167-
string platform = "";
167+
List<string> platforms = new List<string>();
168+
bool checkCUBLAS = false;
168169
switch (buildTarget)
169170
{
170171
case BuildTarget.StandaloneWindows:
171172
case BuildTarget.StandaloneWindows64:
172-
platform = "windows";
173+
platforms.Add("win-x64");
174+
checkCUBLAS = true;
173175
break;
174176
case BuildTarget.StandaloneLinux64:
175-
platform = "linux";
177+
platforms.Add("linux-x64");
178+
checkCUBLAS = true;
176179
break;
177180
case BuildTarget.StandaloneOSX:
178-
platform = "macos";
181+
platforms.Add("osx-universal");
179182
break;
180183
case BuildTarget.Android:
181-
platform = "android";
184+
platforms.Add("android-arm64");
185+
platforms.Add("android-x64");
182186
break;
183187
case BuildTarget.iOS:
184-
platform = "ios";
188+
platforms.Add("ios-arm64");
185189
break;
186190
case BuildTarget.VisionOS:
187-
platform = "visionos";
191+
platforms.Add("visionos-arm64");
188192
break;
189193
}
190194

191195
foreach (string source in Directory.GetDirectories(LLMUnitySetup.libraryPath))
192196
{
193197
string sourceName = Path.GetFileName(source);
194-
bool move = !sourceName.StartsWith(platform);
195-
move = move || (sourceName.Contains("cuda") && !sourceName.Contains("full") && LLMUnitySetup.FullLlamaLib);
196-
move = move || (sourceName.Contains("cuda") && sourceName.Contains("full") && !LLMUnitySetup.FullLlamaLib);
197-
if (move)
198+
if (!platforms.Contains(sourceName))
198199
{
199200
string target = Path.Combine(BuildTempDir, sourceName);
200201
MoveAction(source, target);
201202
MoveAction(source + ".meta", target + ".meta");
202203
}
203204
}
204205

206+
if (checkCUBLAS)
207+
{
208+
List<string> exclusionKeywords = LLMUnitySetup.CUBLAS ? new List<string>() { "tinyblas" } : new List<string>() { "cublas", "cudart" };
209+
foreach (string platform in platforms)
210+
{
211+
string platformDir = Path.Combine(LLMUnitySetup.libraryPath, platform, "native");
212+
foreach (string source in Directory.GetFiles(platformDir))
213+
{
214+
string sourceName = Path.GetFileName(source);
215+
foreach (string exclusionKeyword in exclusionKeywords)
216+
{
217+
if (sourceName.Contains(exclusionKeyword))
218+
{
219+
string target = Path.Combine(BuildTempDir, platform, "native", sourceName);
220+
MoveAction(source, target);
221+
MoveAction(source + ".meta", target + ".meta"); ;
222+
break;
223+
}
224+
}
225+
}
226+
}
227+
}
228+
205229
if (buildTarget == BuildTarget.Android || buildTarget == BuildTarget.iOS || buildTarget == BuildTarget.VisionOS)
206230
{
207-
string source = Path.Combine(LLMUnitySetup.libraryPath, platform);
208-
string target = PluginLibraryDir(buildTarget.ToString());
209-
string pluginDir = PluginDir(buildTarget.ToString());
210-
MoveAction(source, target);
211-
MoveAction(source + ".meta", target + ".meta");
212-
AddActionAddMeta(pluginDir);
231+
foreach (string platform in platforms)
232+
{
233+
string source = Path.Combine(LLMUnitySetup.libraryPath, platform, "native");
234+
string target = Path.Combine(PluginLibraryDir(buildTarget.ToString()), platform);
235+
string pluginDir = PluginDir(buildTarget.ToString());
236+
MoveAction(source, target);
237+
MoveAction(source + ".meta", target + ".meta");
238+
AddActionAddMeta(pluginDir);
239+
}
213240
}
214241
}
215242

216243
static void OnPostprocessAllAssets(string[] importedAssets, string[] deletedAssets, string[] movedAssets, string[] movedFromAssetPaths, bool didDomainReload)
217244
{
218-
foreach (BuildTarget buildTarget in new BuildTarget[]{BuildTarget.iOS, BuildTarget.VisionOS})
245+
foreach (BuildTarget buildTarget in new BuildTarget[] { BuildTarget.iOS, BuildTarget.VisionOS, BuildTarget.Android })
219246
{
220-
string pathToPlugin = Path.Combine("Assets", PluginLibraryDir(buildTarget.ToString(), true), $"libundreamai_{buildTarget.ToString().ToLower()}.a");
221-
for (int i = 0; i < movedAssets.Length; i++)
247+
string suffix = (buildTarget == BuildTarget.Android) ? "so" : "a";
248+
foreach (string platformDir in Directory.GetDirectories(Path.Combine("Assets", PluginLibraryDir(buildTarget.ToString(), true))))
222249
{
223-
if (movedAssets[i] == pathToPlugin)
250+
string platform = Path.GetFileName(platformDir);
251+
string pathToPlugin = Path.Combine(platformDir, $"libllamalib_{platform}.{suffix}");
252+
for (int i = 0; i < movedAssets.Length; i++)
224253
{
225-
var importer = AssetImporter.GetAtPath(pathToPlugin) as PluginImporter;
226-
if (importer != null && importer.isNativePlugin)
254+
if (movedAssets[i] == pathToPlugin)
227255
{
228-
importer.SetCompatibleWithPlatform(buildTarget, true);
229-
importer.SetPlatformData(buildTarget, "CPU", "ARM64");
230-
AssetDatabase.ImportAsset(pathToPlugin);
256+
var importer = AssetImporter.GetAtPath(pathToPlugin) as PluginImporter;
257+
if (importer != null && importer.isNativePlugin)
258+
{
259+
importer.SetCompatibleWithPlatform(buildTarget, true);
260+
importer.SetPlatformData(buildTarget, "CPU", platform.Split("_")[1].ToUpper());
261+
AssetDatabase.ImportAsset(pathToPlugin);
262+
}
231263
}
232264
}
233265
}

‎Runtime/LLMLib.cs‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -508,11 +508,11 @@ public static string GetArchitectureCheckerPath()
508508
string filename;
509509
if (Application.platform == RuntimePlatform.WindowsEditor || Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsServer)
510510
{
511-
filename = $"windows-archchecker/archchecker.dll";
511+
filename = "win-x64/native/llamalib_win-x64_runtime.dll";
512512
}
513513
else if (Application.platform == RuntimePlatform.LinuxEditor || Application.platform == RuntimePlatform.LinuxPlayer || Application.platform == RuntimePlatform.LinuxServer)
514514
{
515-
filename = $"linux-archchecker/libarchchecker.so";
515+
filename = "linux-x64/native/libllamalib_linux-x64_runtime.so";
516516
}
517517
else
518518
{
@@ -529,24 +529,24 @@ public static string GetArchitectureCheckerPath()
529529
public static List<string> GetArchitectureDependencies(string arch)
530530
{
531531
List<string> dependencies = new List<string>();
532-
if (arch == "cuda-cu12.2.0-full")
532+
if (arch == "cublas")
533533
{
534534
if (Application.platform == RuntimePlatform.WindowsEditor || Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsServer)
535535
{
536-
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, $"windows-{arch}/cudart64_12.dll"));
537-
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, $"windows-{arch}/cublasLt64_12.dll"));
538-
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, $"windows-{arch}/cublas64_12.dll"));
536+
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, "win-x64/native/cudart64_12.dll"));
537+
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, "win-x64/native/cublasLt64_12.dll"));
538+
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, "win-x64/native/cublas64_12.dll"));
539539
}
540540
}
541541
else if (arch == "vulkan")
542542
{
543543
if (Application.platform == RuntimePlatform.WindowsEditor || Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsServer)
544544
{
545-
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, $"windows-{arch}/vulkan-1.dll"));
545+
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, "win-x64/native/vulkan-1.dll"));
546546
}
547547
else if (Application.platform == RuntimePlatform.LinuxEditor || Application.platform == RuntimePlatform.LinuxPlayer || Application.platform == RuntimePlatform.LinuxServer)
548548
{
549-
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, $"linux-{arch}/libvulkan.so.1"));
549+
dependencies.Add(Path.Combine(LLMUnitySetup.libraryPath, "linux-x64/native/libvulkan.so.1"));
550550
}
551551
}
552552
return dependencies;
@@ -560,17 +560,18 @@ public static List<string> GetArchitectureDependencies(string arch)
560560
public static string GetArchitecturePath(string arch)
561561
{
562562
string filename;
563+
string os = OSName();
563564
if (Application.platform == RuntimePlatform.WindowsEditor || Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsServer)
564565
{
565-
filename = $"windows-{arch}/undreamai_windows-{arch}.dll";
566+
filename = $"{os}/native/llamalib_${os}_{arch}.dll";
566567
}
567568
else if (Application.platform == RuntimePlatform.LinuxEditor || Application.platform == RuntimePlatform.LinuxPlayer || Application.platform == RuntimePlatform.LinuxServer)
568569
{
569-
filename = $"linux-{arch}/libundreamai_linux-{arch}.so";
570+
filename = $"{os}/native/libllamalib_${os}_{arch}.so";
570571
}
571572
else if (Application.platform == RuntimePlatform.OSXEditor || Application.platform == RuntimePlatform.OSXPlayer || Application.platform == RuntimePlatform.OSXServer)
572573
{
573-
filename = $"macos-{arch}/libundreamai_macos-{arch}.dylib";
574+
filename = $"{os}/native/libllamalib_${os}_{arch}.dylib";
574575
}
575576
else
576577
{
@@ -634,6 +635,18 @@ public static string GetArchitecturePath(string arch)
634635

635636
#endif
636637

638+
public static string OSName()
639+
{
640+
if (Application.platform == RuntimePlatform.WindowsEditor || Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsServer)
641+
return "win-x64";
642+
else if (Application.platform == RuntimePlatform.LinuxEditor || Application.platform == RuntimePlatform.LinuxPlayer || Application.platform == RuntimePlatform.LinuxServer)
643+
return "linux-x64";
644+
else if (Application.platform == RuntimePlatform.OSXEditor || Application.platform == RuntimePlatform.OSXPlayer)
645+
return "osx-universal";
646+
string error = "Unknown OS";
647+
LLMUnitySetup.LogError(error);
648+
throw new Exception(error);
649+
}
637650
/// <summary>
638651
/// Identifies the possible architectures that we can use based on the OS and GPU usage
639652
/// </summary>
@@ -647,14 +660,8 @@ public static List<string> PossibleArchitectures(bool gpu = false)
647660
{
648661
if (gpu)
649662
{
650-
if (LLMUnitySetup.FullLlamaLib)
651-
{
652-
architectures.Add("cuda-cu12.2.0-full");
653-
}
654-
else
655-
{
656-
architectures.Add("cuda-cu12.2.0");
657-
}
663+
architectures.Add("cublas");
664+
architectures.Add("tinyblas");
658665
architectures.Add("hip");
659666
architectures.Add("vulkan");
660667
}

0 commit comments

Comments
 (0)