Merge pull request #708 from ionite34/backport/main/pr-707

[dev to main] backport: Fix exceptions in litedb deserialize for ModelIndexService (707)

Author: ionite34

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2
 - Fooocus Package - Added `pip>=23.3.2,<24.1` specifier before install, fixes potential install errors due to deprecated requirement spec used by `torchsde`.
 - Fixed error when launching SwarmUI when installed to a path with spaces
 - Fixed issue where model folders were being created too late in certain cases
+- Fixed [#683](https://github.com/LykosAI/StabilityMatrix/issues/683) - Model indexing causing LiteDB errors after upgrading from older versions due to updated enum values
 ### Supporters
 #### Visionaries
 - Huge thanks to our Visionary-tier supporters on Patreon, **Scopp Mcdee** and **Waterclouds**! Your support helps us continue to improve Stability Matrix!

StabilityMatrix.Core/Database/ILiteDbContext.cs

@@ -21,4 +21,9 @@ public interface ILiteDbContext : IDisposable
     Task<bool> UpsertCivitModelQueryCacheEntryAsync(CivitModelQueryCacheEntry entry);
     Task<GithubCacheEntry?> GetGithubCacheEntry(string cacheKey);
     Task<bool> UpsertGithubCacheEntry(GithubCacheEntry cacheEntry);
+
+    /// <summary>
+    /// Clear all Collections that store re-fetchable cache type data.
+    /// </summary>
+    Task ClearAllCacheCollectionsAsync();
 }

StabilityMatrix.Core/Database/LiteDbContext.cs

@@ -174,6 +174,30 @@ public async Task<bool> UpsertCivitModelQueryCacheEntryAsync(CivitModelQueryCach
     public Task<bool> UpsertGithubCacheEntry(GithubCacheEntry cacheEntry) =>
         GithubCache.UpsertAsync(cacheEntry);
 
+    /// <summary>
+    /// Clear all Collections that store re-fetchable cache type data.
+    /// </summary>
+    public async Task ClearAllCacheCollectionsAsync()
+    {
+        var collectionNames = new List<string>
+        {
+            nameof(CivitModels),
+            nameof(CivitModelVersions),
+            nameof(CivitModelQueryCache),
+            nameof(GithubCache),
+            nameof(LocalModelFiles),
+            nameof(LocalImageFiles)
+        };
+
+        logger.LogInformation("Clearing all cache collections: [{@Names}]", collectionNames);
+
+        foreach (var name in collectionNames)
+        {
+            var collection = Database.GetCollection(name);
+            await collection.DeleteAllAsync().ConfigureAwait(false);
+        }
+    }
+
     public void Dispose()
     {
         if (lazyDatabase.IsValueCreated)

StabilityMatrix.Core/Services/ModelIndexService.cs

@@ -2,10 +2,11 @@
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Text;
-using System.Text.Json;
 using AsyncAwaitBestPractices;
 using AutoCtor;
 using KGySoft.CoreLibraries;
+using LiteDB;
+using LiteDB.Async;
 using Microsoft.Extensions.Logging;
 using StabilityMatrix.Core.Attributes;
 using StabilityMatrix.Core.Database;
@@ -15,6 +16,7 @@
 using StabilityMatrix.Core.Models.Api;
 using StabilityMatrix.Core.Models.Database;
 using StabilityMatrix.Core.Models.FileInterfaces;
+using JsonSerializer = System.Text.Json.JsonSerializer;
 
 namespace StabilityMatrix.Core.Services;
 
@@ -103,9 +105,34 @@ private async Task LoadFromDbAsync()
 
         logger.LogInformation("Loading models from database...");
 
-        var allModels = (
-            await liteDbContext.LocalModelFiles.IncludeAll().FindAllAsync().ConfigureAwait(false)
-        ).ToImmutableArray();
+        ImmutableArray<LocalModelFile> allModels = [];
+        try
+        {
+            allModels =
+            [
+                ..(
+                    await liteDbContext.LocalModelFiles.IncludeAll().FindAllAsync().ConfigureAwait(false)
+                )
+            ];
+        }
+        catch (Exception e)
+        {
+            // Handle enum deserialize exceptions from changes
+            if (e is LiteException or LiteAsyncException && e.InnerException is ArgumentException inner)
+            {
+                logger.LogWarning(
+                    e,
+                    "LiteDb Deserialize error while fetching LocalModelFiles '{Inner}', cache collections will be cleared",
+                    inner.ToString()
+                );
+
+                await liteDbContext.ClearAllCacheCollectionsAsync().ConfigureAwait(false);
+            }
+            else
+            {
+                throw;
+            }
+        }
 
         ModelIndex = allModels.GroupBy(m => m.SharedFolderType).ToDictionary(g => g.Key, g => g.ToList());
 
@@ -479,12 +506,34 @@ private async Task RefreshIndexParallelCore()
 
             if (model.LatestModelInfo == null && model.HasConnectedModel)
             {
-                var civitModel = await liteDbContext
-                    .CivitModels.Include(m => m.ModelVersions)
-                    .FindByIdAsync(model.ConnectedModelInfo.ModelId)
-                    .ConfigureAwait(false);
+                try
+                {
+                    model.LatestModelInfo = await liteDbContext
+                        .CivitModels.Include(m => m.ModelVersions)
+                        .FindByIdAsync(model.ConnectedModelInfo.ModelId)
+                        .ConfigureAwait(false);
+                }
+                catch (Exception e)
+                {
+                    // Handle enum deserialize exceptions from changes
+                    if (
+                        e is LiteException or LiteAsyncException
+                        && e.InnerException is ArgumentException inner
+                    )
+                    {
+                        logger.LogWarning(
+                            e,
+                            "LiteDb Deserialize error while fetching CivitModels '{Inner}', cache collections will be cleared",
+                            inner.ToString()
+                        );
 
-                model.LatestModelInfo = civitModel;
+                        await liteDbContext.ClearAllCacheCollectionsAsync().ConfigureAwait(false);
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
             }
             var list = newIndex.GetOrAdd(model.SharedFolderType);
             list.Add(model);