Merge pull request microsoft#351 from microsoft/zt/custom-models

[DRAFT] Uploading custom models for any sample

Author: nmetulev

AIDevGallery.SourceGenerator/Models/Model.cs

@@ -27,4 +27,6 @@ internal class Model
     public List<AIToolkitAction>? AIToolkitActions { get; init; }
     public string? AIToolkitId { get; init; }
     public string? AIToolkitFinetuningId { get; init; }
+    public List<int[]>? InputDimensions { get; set; }
+    public List<int[]>? OutputDimensions { get; set; }
 }

AIDevGallery.SourceGenerator/ModelsSourceGenerator.cs

@@ -253,6 +253,8 @@ private void GenerateModelDetails(StringBuilder sourceBuilder, Dictionary<string
             var aiToolkitActions = modelDefinition.AIToolkitActions != null ? string.Join(", ", modelDefinition.AIToolkitActions.Select(action => $"AIToolkitAction.{action}")) : string.Empty;
             var aiToolkitId = !string.IsNullOrEmpty(modelDefinition.AIToolkitId) ? $"\"{modelDefinition.AIToolkitId}\"" : "null";
             var aiToolkitFinetuningId = !string.IsNullOrEmpty(modelDefinition.AIToolkitFinetuningId) ? $"\"{modelDefinition.AIToolkitFinetuningId}\"" : "null";
+            var inputDimensions = modelDefinition.InputDimensions != null ? "[ " + string.Join(", ", modelDefinition.InputDimensions.Select(dimension => "[" + string.Join(", ", dimension.Select(d => d.ToString())) + "]")) + "]" : "null";
+            var outputDimensions = modelDefinition.OutputDimensions != null ? "[ " + string.Join(", ", modelDefinition.OutputDimensions.Select(dimension => "[" + string.Join(", ", dimension.Select(d => d.ToString())) + "]")) + "]" : "null";
 
             sourceBuilder.AppendLine(
                 $$""""
@@ -274,7 +276,9 @@ private void GenerateModelDetails(StringBuilder sourceBuilder, Dictionary<string
                                 FileFilters = [ {{fileFilters}} ],
                                 AIToolkitActions = [ {{aiToolkitActions}} ],
                                 AIToolkitId = {{aiToolkitId}},
-                                AIToolkitFinetuningId = {{aiToolkitFinetuningId}}
+                                AIToolkitFinetuningId = {{aiToolkitFinetuningId}},
+                                InputDimensions = {{inputDimensions}},
+                                OutputDimensions = {{outputDimensions}}
                             }
                         },
                 """");

AIDevGallery/AIDevGallery.csproj

@@ -84,9 +84,9 @@
     <PackageReference Include="NAudio.WinMM" />
     <PackageReference Include="System.Numerics.Tensors" />
     <PackageReference Include="WinUIEx" />
-    <PackageReference Include="HtmlAgilityPack" />
-    <PackageReference Include="Markdig" />
-    <PackageReference Include="Roman-Numerals" />
+    <PackageReference Include="HtmlAgilityPack"/>
+    <PackageReference Include="Markdig"/>
+    <PackageReference Include="Roman-Numerals"/>
     <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Managed" />
     <Manifest Include="$(ApplicationManifest)" />
   </ItemGroup>

AIDevGallery/App.xaml.cs

@@ -41,7 +41,7 @@ protected override async void OnLaunched(LaunchActivatedEventArgs args)
     {
         await LoadSamples();
         AppActivationArguments appActivationArguments = AppInstance.GetCurrent().GetActivatedEventArgs();
-        var activationParam = ActivationHelper.GetActivationParam(appActivationArguments);
+        var activationParam = await ActivationHelper.GetActivationParam(appActivationArguments);
         MainWindow = new MainWindow(activationParam);
 
         MainWindow.Activate();

AIDevGallery/Controls/ModelPicker/AddHFModelView.xaml.cs

@@ -99,7 +99,7 @@ private async Task SearchModels(string query, CancellationToken cancellationToke
 
                 try
                 {
-                    accelerator = UserAddedModelUtilsTemp.GetHardwareAcceleratorFromConfig(configContents);
+                    accelerator = UserAddedModelUtil.GetHardwareAcceleratorFromConfig(configContents);
                 }
                 catch (JsonException)
                 {

AIDevGallery/Controls/ModelPicker/ModelPickerViews/OnnxPickerView.xaml.cs

@@ -16,13 +16,13 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Windows.ApplicationModel.DataTransfer;
-using Windows.Storage.Pickers;
 
 namespace AIDevGallery.Controls.ModelPickerViews;
 internal sealed partial class OnnxPickerView : BaseModelPickerView
 {
-    private List<ModelDetails> Models { get; set; } = new();
-    private List<ModelType>? ModelTypes { get; set; }
+    private List<ModelDetails> models = [];
+    private List<ModelType>? modelTypes;
+
     public ModelDetails? Selected { get; private set; }
 
     private ObservableCollection<AvailableModel> AvailableModels { get; } = [];
@@ -38,48 +38,47 @@ public OnnxPickerView()
 
     public override Task Load(List<ModelType> types)
     {
-        ModelTypes = types;
+        modelTypes = types;
+
+        ResetAndLoadModelList();
 
         if (types.Contains(ModelType.LanguageModels))
         {
             AddHFModelButton.Visibility = Visibility.Visible;
         }
 
         // local models supported for types
-        // TODO: check which models support it
-        if (types.Contains(ModelType.LanguageModels) || true)
+        if (types.Contains(ModelType.LanguageModels) || models.IsModelsDetailsListUploadCompatible())
         {
             AddLocalModelButton.Visibility = Visibility.Visible;
         }
 
-        ResetAndLoadModelList();
-
         return Task.CompletedTask;
     }
 
     private void ResetAndLoadModelList()
     {
-        Models.Clear();
+        models.Clear();
         AvailableModels.Clear();
         DownloadableModels.Clear();
         UnavailableModels.Clear();
 
-        if (ModelTypes == null || ModelTypes.Count == 0)
+        if (modelTypes == null || modelTypes.Count == 0)
         {
             return;
         }
 
-        foreach (ModelType type in ModelTypes)
+        foreach (ModelType type in modelTypes)
         {
-            Models.AddRange(ModelDetailsHelper.GetModelDetailsForModelType(type));
+            models.AddRange(ModelDetailsHelper.GetModelDetailsForModelType(type));
         }
 
-        if (Models == null || Models.Count == 0)
+        if (models == null || models.Count == 0)
         {
             return;
         }
 
-        foreach (var model in Models)
+        foreach (var model in models)
         {
             if (!model.IsOnnxModel())
             {
@@ -155,7 +154,7 @@ private void ResetAndLoadModelList()
 
     private void CacheStore_ModelsChanged(ModelCacheStore sender)
     {
-        ResetAndLoadModelList();
+        DispatcherQueue.TryEnqueue(ResetAndLoadModelList);
     }
 
     private void ModelSelectionView_SelectionChanged(object sender, SelectionChangedEventArgs e)
@@ -269,6 +268,7 @@ private async void DeleteModel_Click(object sender, RoutedEventArgs e)
             if (result == ContentDialogResult.Primary)
             {
                 await App.ModelCache.DeleteModelFromCache(details.Url);
+                ResetAndLoadModelList();
             }
         }
     }
@@ -375,129 +375,20 @@ private void AddHFModelView_CloseRequested(object sender)
 
     private async void AddLocalModelButton_Click(object sender, RoutedEventArgs e)
     {
-        var hwnd = WinRT.Interop.WindowNative.GetWindowHandle(App.MainWindow);
-        var picker = new FolderPicker();
-        picker.FileTypeFilter.Add("*");
-        WinRT.Interop.InitializeWithWindow.Initialize(picker, hwnd);
-        var folder = await picker.PickSingleFolderAsync();
-
-        if (folder != null)
+        if (modelTypes == null)
         {
-            var files = Directory.GetFiles(folder.Path);
-            var config = files.Where(r => Path.GetFileName(r) == "genai_config.json").FirstOrDefault();
-
-            if (string.IsNullOrEmpty(config) || App.ModelCache.Models.Any(m => m.Path == folder.Path))
-            {
-                var message = string.IsNullOrEmpty(config) ?
-                    "The folder does not contain a model you can add. Ensure \"genai_config.json\" is present in the selected directory" :
-                    "This model is already added";
-
-                ContentDialog confirmFolderDialog = new()
-                {
-                    Title = "Can't add model",
-                    Content = message,
-                    XamlRoot = this.Content.XamlRoot,
-                    CloseButtonText = "OK"
-                };
-
-                await confirmFolderDialog.ShowAsync();
-                return;
-            }
-
-            HardwareAccelerator accelerator = HardwareAccelerator.CPU;
-
-            try
-            {
-                string configContents = string.Empty;
-                configContents = await File.ReadAllTextAsync(config);
-                accelerator = UserAddedModelUtilsTemp.GetHardwareAcceleratorFromConfig(configContents);
-            }
-            catch (Exception ex)
-            {
-                ContentDialog confirmFolderDialog = new()
-                {
-                    Title = "Can't read genai_config.json",
-                    Content = ex.Message,
-                    XamlRoot = this.Content.XamlRoot,
-                    CloseButtonText = "OK"
-                };
-
-                await confirmFolderDialog.ShowAsync();
-                return;
-            }
-
-            var nameTextBox = new TextBox()
-            {
-                Text = Path.GetFileName(folder.Path),
-                Width = 300,
-                HorizontalAlignment = HorizontalAlignment.Left,
-                Margin = new Thickness(0, 0, 0, 10),
-                Header = "Model name"
-            };
-
-            ContentDialog nameModelDialog = new()
-            {
-                Title = "Add model",
-                Content = new StackPanel()
-                {
-                    Orientation = Orientation.Vertical,
-                    Spacing = 8,
-                    Children =
-                    {
-                        new TextBlock()
-                        {
-                            Text = $"Adding ONNX model from \n \"{folder.Path}\"",
-                            TextWrapping = TextWrapping.WrapWholeWords
-                        },
-                        nameTextBox
-                    }
-                },
-                XamlRoot = this.Content.XamlRoot,
-                CloseButtonText = "Cancel",
-                PrimaryButtonText = "Add",
-                DefaultButton = ContentDialogButton.Primary,
-                Style = Application.Current.Resources["DefaultContentDialogStyle"] as Style
-            };
-
-            string modelName = nameTextBox.Text;
-
-            nameTextBox.TextChanged += (s, e) =>
-            {
-                if (string.IsNullOrEmpty(nameTextBox.Text))
-                {
-                    nameModelDialog.IsPrimaryButtonEnabled = false;
-                }
-                else
-                {
-                    modelName = nameTextBox.Text;
-                    nameModelDialog.IsPrimaryButtonEnabled = true;
-                }
-            };
-
-            var result = await nameModelDialog.ShowAsync();
-            if (result != ContentDialogResult.Primary)
-            {
-                return;
-            }
-
-            DirectoryInfo dirInfo = new DirectoryInfo(folder.Path);
-            long dirSize = await Task.Run(() => dirInfo.EnumerateFiles("*", SearchOption.AllDirectories).Sum(file => file.Length));
-
-            var details = new ModelDetails()
-            {
-                Id = "useradded-local-languagemodel-" + Guid.NewGuid().ToString(),
-                Name = modelName,
-                Url = $"local-file:///{folder.Path}",
-                Description = "Localy added GenAI Model",
-                HardwareAccelerators = [accelerator],
-                IsUserAdded = true,
-                PromptTemplate = ModelDetailsHelper.GetTemplateFromName(folder.Path),
-                Size = dirSize,
-                ReadmeUrl = null,
-                License = "unknown"
-            };
+            return;
+        }
 
-            await App.ModelCache.AddLocalModelToCache(details, folder.Path);
+        if (modelTypes.Contains(ModelType.LanguageModels))
+        {
+            await UserAddedModelUtil.OpenAddLanguageModelFlow(Content.XamlRoot);
+        }
+        else
+        {
+            await UserAddedModelUtil.OpenAddModelFlow(Content.XamlRoot, modelTypes);
         }
+
+        ResetAndLoadModelList();
     }
 }

AIDevGallery/Controls/ModelSelectionControl.xaml

@@ -612,6 +612,11 @@
                         </DataTemplate>
                     </ItemsRepeater.ItemTemplate>
                 </ItemsRepeater>
+                <!--<Button 
+                    x:Name="AddLocalModelButton"
+                    Click="AddLocalModelButton_Click"
+                    HorizontalAlignment="Center" 
+                    Content="Add Local Model"/>-->
             </StackPanel>
         </ScrollViewer>
         <ContentDialog

AIDevGallery/Controls/ModelSelectionControl.xaml.cs

@@ -22,6 +22,7 @@ namespace AIDevGallery.Controls;
 internal partial class ModelSelectionControl : UserControl
 {
     public List<ModelDetails>? Models { get; private set; }
+    public Scenario? Scenario { get; set; }
     public ModelDetails? Selected { get; private set; }
 
     public static readonly DependencyProperty DownloadableModelsTitleProperty = DependencyProperty.Register(nameof(DownloadableModelsTitle), typeof(string), typeof(ModelSelectionControl), new PropertyMetadata(defaultValue: null));
@@ -85,7 +86,7 @@ public void SetModels(List<ModelDetails>? models, ModelDetails? initialSelectedM
 
     private void CacheStore_ModelsChanged(ModelCacheStore sender)
     {
-        PopulateModelDetailsLists();
+        DispatcherQueue.TryEnqueue(PopulateModelDetailsLists);
     }
 
     private void ResetAndLoadModelList(ModelDetails? selectedModel = null)