Merge pull request #3203 from Nexus-Mods/main

Backport fixes from main

Author: Al12rs

Directory.Packages.props

@@ -16,10 +16,10 @@
     <PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="9.4.0" />
     <PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
     <PackageVersion Include="Nerdbank.Streams" Version="2.11.79" />
-    <PackageVersion Include="NexusMods.Cascade" Version="0.11.0" />
-    <PackageVersion Include="NexusMods.Cascade.SourceGenerator" Version="0.11.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.12.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.12.0" />
+    <PackageVersion Include="NexusMods.Cascade" Version="0.12.0" />
+    <PackageVersion Include="NexusMods.Cascade.SourceGenerator" Version="0.12.0" />
+    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.13.0" />
+    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.13.0" />
     <PackageVersion Include="NexusMods.Hashing.xxHash3.Paths" Version="3.0.3" />
     <PackageVersion Include="NexusMods.Hashing.xxHash3" Version="3.0.3" />
     <PackageVersion Include="NexusMods.Archives.Nx" Version="0.6.4" />
@@ -149,7 +149,7 @@
     <PackageVersion Include="Humanizer" Version="2.14.1" />
     <PackageVersion Include="ini-parser-netstandard" Version="2.5.2" />
     <PackageVersion Include="Mutagen.Bethesda.Skyrim" Version="0.44.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.12.0" />
+    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.13.0" />
     <PackageVersion Include="NLog.Extensions.Logging" Version="5.3.14" />
     <PackageVersion Include="OneOf" Version="3.0.271" />
     <PackageVersion Include="ReactiveUI.Fody" Version="19.5.41" />

src/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -353,7 +353,7 @@ private static bool FileIsEnabled(LoadoutItem.ReadOnly arg)
     /// <inheritdoc />
     public async Task<Dictionary<GamePath, SyncNode>> BuildSyncTree(Loadout.ReadOnly loadout)
     {
-        var metadata = await ReindexState(loadout.InstallationInstance, Connection);
+        var metadata = await ReindexState(loadout.InstallationInstance, false, Connection);
         var previouslyApplied = loadout.Installation.GetLastAppliedDiskState();
         return BuildSyncTree(DiskStateToPathPartPair(metadata.DiskStateEntries), DiskStateToPathPartPair(previouslyApplied), loadout);
     }
@@ -908,11 +908,11 @@ private bool ActionIngestFromDisk(Dictionary<GamePath, SyncNode> syncTree, Loado
         return await RunActions(tree, loadout);
     }
 
-    public async Task<GameInstallMetadata.ReadOnly> RescanFiles(GameInstallation gameInstallation)
+    public async Task<GameInstallMetadata.ReadOnly> RescanFiles(GameInstallation gameInstallation, bool ignoreModifiedDates)
     {
         // Make sure the file hashes are up to date
         await _fileHashService.GetFileHashesDb();
-        return await ReindexState(gameInstallation, Connection);
+        return await ReindexState(gameInstallation, ignoreModifiedDates, Connection);
     }
 
     /// <summary>
@@ -1201,14 +1201,14 @@ await Parallel.ForEachAsync(files, async (item, _) =>
     /// <summary>
     /// Reindex the state of the game, running a transaction if changes are found
     /// </summary>
-    private async Task<GameInstallMetadata.ReadOnly> ReindexState(GameInstallation installation, IConnection connection)
+    private async Task<GameInstallMetadata.ReadOnly> ReindexState(GameInstallation installation, bool ignoreModifiedDates, IConnection connection)
     {        
         using var _ = await _lock.LockAsync();
         var originalMetadata = installation.GetMetadata(connection);
         using var tx = connection.BeginTransaction();
 
         // Index the state
-        var changed = await ReindexState(installation, connection, tx);
+        var changed = await ReindexState(installation, ignoreModifiedDates, connection, tx);
 
         if (!originalMetadata.Contains(GameInstallMetadata.InitialDiskStateTransaction))
         {
@@ -1219,6 +1219,7 @@ await Parallel.ForEachAsync(files, async (item, _) =>
 
         if (changed)
         {
+            tx.Add(installation.GameMetadataId, GameInstallMetadata.LastScannedDiskStateTransactionId, EntityId.From(TxId.Tmp.Value));
             await tx.Commit();
         }
 
@@ -1228,7 +1229,7 @@ await Parallel.ForEachAsync(files, async (item, _) =>
     /// <summary>
     /// Reindex the state of the game
     /// </summary>
-    public async Task<bool> ReindexState(GameInstallation installation, IConnection connection, ITransaction tx)
+    public async Task<bool> ReindexState(GameInstallation installation, bool ignoreModifiedDates, IConnection connection, ITransaction tx)
     {
         var hashDb = await _fileHashService.GetFileHashesDb();
 
@@ -1262,50 +1263,61 @@ public async Task<bool> ReindexState(GameInstallation installation, IConnection
 
             await Parallel.ForEachAsync(locationPath.EnumerateFiles(), async (file, token) =>
             {
-                var gamePath = installation.LocationsRegister.ToGamePath(file);
-                if (ShouldIgnorePathWhenIndexing(gamePath)) return;
-
-                bool isNewPath;
-                lock (seenPathsLock)
+                try
                 {
-                    isNewPath = seenPaths.Add(gamePath);
-                }
+                    var gamePath = installation.LocationsRegister.ToGamePath(file);
+                    if (ShouldIgnorePathWhenIndexing(gamePath)) return;
 
-                if (!isNewPath)
-                {
-                    Logger.LogDebug("Skipping already indexed file at `{Path}`", file);
-                    return;
-                }
+                    bool isNewPath;
+                    lock (seenPathsLock)
+                    {
+                        isNewPath = seenPaths.Add(gamePath);
+                    }
 
-                if (previousDiskState.TryGetValue(gamePath, out var previousDiskStateEntry))
-                {
-                    var fileInfo = file.FileInfo;
-                    var writeTimeUtc = new DateTimeOffset(fileInfo.LastWriteTimeUtc);
+                    if (!isNewPath)
+                    {
+                        Logger.LogDebug("Skipping already indexed file at `{Path}`", file);
+                        return;
+                    }
+
+                    if (previousDiskState.TryGetValue(gamePath, out var previousDiskStateEntry))
+                    {
+                        var fileInfo = file.FileInfo;
+                        var writeTimeUtc = new DateTimeOffset(fileInfo.LastWriteTimeUtc);
 
-                    // If the files don't match, update the entry
-                    if (writeTimeUtc != previousDiskStateEntry.LastModified || fileInfo.Size != previousDiskStateEntry.Size)
+                        // If the files don't match, update the entry
+                        if (writeTimeUtc != previousDiskStateEntry.LastModified || fileInfo.Size != previousDiskStateEntry.Size || ignoreModifiedDates)
+                        {
+                            var newHash = await MaybeHashFile(hashDb, gamePath, file,
+                                fileInfo, token
+                            );
+                            tx.Add(previousDiskStateEntry.Id, DiskStateEntry.Size, fileInfo.Size);
+                            tx.Add(previousDiskStateEntry.Id, DiskStateEntry.Hash, newHash);
+                            tx.Add(previousDiskStateEntry.Id, DiskStateEntry.LastModified, writeTimeUtc);
+                            hasDiskStateChanged = true;
+                        }
+                    }
+                    else
                     {
-                        var newHash = await MaybeHashFile(hashDb, gamePath, file, fileInfo, token);
-                        tx.Add(previousDiskStateEntry.Id, DiskStateEntry.Size, fileInfo.Size);
-                        tx.Add(previousDiskStateEntry.Id, DiskStateEntry.Hash, newHash);
-                        tx.Add(previousDiskStateEntry.Id, DiskStateEntry.LastModified, writeTimeUtc);
+                        var newHash = await MaybeHashFile(hashDb, gamePath, file,
+                            file.FileInfo, token
+                        );
+
+                        _ = new DiskStateEntry.New(tx, tx.TempId(DiskStateEntry.EntryPartition))
+                        {
+                            Path = gamePath.ToGamePathParentTuple(gameInstallMetadata.Id),
+                            Hash = newHash,
+                            Size = file.FileInfo.Size,
+                            LastModified = file.FileInfo.LastWriteTimeUtc,
+                            GameId = gameInstallMetadata.Id,
+                        };
+
                         hasDiskStateChanged = true;
                     }
                 }
-                else
+                catch (Exception ex)
                 {
-                    var newHash = await MaybeHashFile(hashDb, gamePath, file, file.FileInfo, token);
-
-                    _ = new DiskStateEntry.New(tx, tx.TempId(DiskStateEntry.EntryPartition))
-                    {
-                        Path = gamePath.ToGamePathParentTuple(gameInstallMetadata.Id),
-                        Hash = newHash,
-                        Size = file.FileInfo.Size,
-                        LastModified = file.FileInfo.LastWriteTimeUtc,
-                        GameId = gameInstallMetadata.Id,
-                    };
-
-                    hasDiskStateChanged = true;
+                    throw ex;
                 }
             });
         }
@@ -1325,6 +1337,9 @@ await Parallel.ForEachAsync(locationPath.EnumerateFiles(), async (file, token) =
     private async ValueTask<Hash> MaybeHashFile(IDb hashDb, GamePath gamePath, AbsolutePath file, IFileEntry fileInfo, CancellationToken token)
     {
         Hash? diskMinimalHash = null;
+
+        var foundHash = Hash.Zero;
+        var needFullHash = true;
 
         // Look for all known files that match the path
         foreach (var matchingPath in PathHashRelation.FindByPath(hashDb, gamePath.Path))
@@ -1338,10 +1353,26 @@ private async ValueTask<Hash> MaybeHashFile(IDb hashDb, GamePath gamePath, Absol
             diskMinimalHash ??= await MultiHasher.MinimalHash(file, token);
 
             if (hash.MinimalHash == diskMinimalHash)
-                return hash.XxHash3;
+            {
+                // We previously found a hash that matches the minimal hash, make sure the xxHash3 matches, otherwise we 
+                // have a hash collision
+                if (foundHash != Hash.Zero && foundHash != hash.XxHash3)
+                {
+                    // We have a hash collision, so we need to do a full hash
+                    needFullHash = true;
+                    break;
+                }
+
+                // Store the hash
+                foundHash = hash.XxHash3;
+                needFullHash = false;
+            }
         }
+        
+        if (!needFullHash)
+            return foundHash;
 
-        Logger.LogDebug("Didn't find matching hash data for file `{Path}`, falling back to doing a full hash", file);
+        Logger.LogDebug("Didn't find matching hash data for file `{Path}` or found multiple matches, falling back to doing a full hash", file);
         return await file.XxHash3Async(token: token);
     }
 
@@ -1460,7 +1491,7 @@ public Optional<LoadoutId> GetCurrentlyActiveLoadout(GameInstallation installati
     public async Task ActivateLoadout(LoadoutId loadoutId)
     {
         var loadout = Loadout.Load(Connection.Db, loadoutId);
-        var reindexed = await ReindexState(loadout.InstallationInstance, Connection);
+        var reindexed = await ReindexState(loadout.InstallationInstance, false, Connection);
 
         var tree = BuildSyncTree(DiskStateToPathPartPair(reindexed.DiskStateEntries), DiskStateToPathPartPair(reindexed.DiskStateEntries), loadout);
         ProcessSyncTree(tree);
@@ -1664,7 +1695,7 @@ public async Task DeleteLoadout(LoadoutId loadoutId, GarbageCollectorRunMode gcR
     public async Task ResetToOriginalGameState(GameInstallation installation, LocatorId[] locatorIds)
     {
         var gameState = _fileHashService.GetGameFiles((installation.Store, locatorIds));
-        var metaData = await ReindexState(installation, Connection);
+        var metaData = await ReindexState(installation, false, Connection);
 
         List<PathPartPair> diskState = [];
 

src/NexusMods.Abstractions.Loadouts.Synchronizers/ILoadoutSynchronizer.cs

@@ -49,7 +49,7 @@ public interface ILoadoutSynchronizer
     /// Rescan the files in the folders this game requires. This is used to bring the local cache up to date with the
     /// whatever is on disk.
     /// </summary>
-    Task<GameInstallMetadata.ReadOnly> RescanFiles(GameInstallation gameInstallation);
+    Task<GameInstallMetadata.ReadOnly> RescanFiles(GameInstallation gameInstallation, bool ignoreModifiedDate = false);
 
     #endregion
 

src/NexusMods.Abstractions.Loadouts.Synchronizers/Rules/ActionMapping.cs

@@ -73,7 +73,7 @@ public static Actions MapActions(SignatureShorthand shorthand)
             AAx_XXx_i => DeleteFromDisk,
             AAx_xxx_I => BackupFile | DeleteFromDisk,
             AAx_XXx_I => DeleteFromDisk,
-            AAA_xxx_i => BackupFile,
+            AAA_xxx_i => DoNothing,
             AAA_XXX_i => DoNothing,
             AAA_xxx_I => DoNothing,
             AAA_XXX_I => DoNothing,

src/NexusMods.Abstractions.NexusWebApi/ILoginManager.cs

@@ -59,7 +59,7 @@ public interface ILoginManager
     /// <summary>
     /// Returns the user's information
     /// </summary>
-    Task<UserInfo?> GetUserInfoAsync(CancellationToken token = default);
+    ValueTask<UserInfo?> GetUserInfoAsync(CancellationToken token = default);
 
     /// <summary>
     /// Verifies whether the user is logged in or not

src/NexusMods.App/Program.cs

@@ -139,7 +139,9 @@ public static int Main(string[] args)
         }
         finally
         {
-            host.StopAsync().Wait(timeout: TimeSpan.FromSeconds(5));
+            // Wait for 15 seconds for the host to stop, otherwise kill the process
+            if (!host.StopAsync().Wait(timeout: TimeSpan.FromSeconds(15)))
+                Environment.Exit(0);
         }
     }
 

src/NexusMods.Collections/CollectionDownloader.cs

@@ -15,6 +15,7 @@
 using NexusMods.Abstractions.NexusWebApi;
 using NexusMods.Abstractions.NexusWebApi.Types;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Abstractions.Settings;
 using NexusMods.Abstractions.Telemetry;
 using NexusMods.CrossPlatform.Process;
 using NexusMods.Extensions.BCL;
@@ -182,7 +183,10 @@ public async ValueTask Download(CollectionDownloadExternal.ReadOnly download, Ca
     /// </summary>
     public async ValueTask Download(CollectionDownloadNexusMods.ReadOnly download, CancellationToken cancellationToken)
     {
-        if (_loginManager.IsPremium)
+        var userInfo = await _loginManager.GetUserInfoAsync(cancellationToken);
+        if (userInfo is null) return;
+
+        if (userInfo.UserRole is UserRole.Premium)
         {
             await using var tempPath = _temporaryFileManager.CreateFile();
             var job = await _nexusModsLibrary.CreateDownloadJob(tempPath, download.FileMetadata, cancellationToken: cancellationToken);
@@ -253,7 +257,6 @@ public async ValueTask DownloadItems(
         CollectionRevisionMetadata.ReadOnly revisionMetadata,
         ItemType itemType,
         IDb db,
-        int maxDegreeOfParallelism = -1,
         CancellationToken cancellationToken = default)
     {
         var job = new DownloadCollectionJob
@@ -263,7 +266,7 @@ public async ValueTask DownloadItems(
             RevisionMetadata = revisionMetadata,
             Db = db,
             ItemType = itemType,
-            MaxDegreeOfParallelism = maxDegreeOfParallelism,
+            MaxDegreeOfParallelism = _serviceProvider.GetRequiredService<ISettingsManager>().Get<DownloadSettings>().MaxParallelDownloads,
         };
 
         await _jobMonitor.Begin<DownloadCollectionJob, R3.Unit>(job);

src/NexusMods.Collections/DownloadCollectionJob.cs

@@ -12,7 +12,7 @@ public class DownloadCollectionJob : IJobDefinitionWithStart<DownloadCollectionJ
     public required CollectionDownloader.ItemType ItemType { get; init; }
     public required CollectionDownloader Downloader { get; init; }
     public required IDb Db { get; init; }
-    public int MaxDegreeOfParallelism { get; init; } = -1;
+    public required int MaxDegreeOfParallelism { get; init; }
 
     public async ValueTask<R3.Unit> StartAsync(IJobContext<DownloadCollectionJob> context)
     {

src/NexusMods.Collections/DownloadSettings.cs

@@ -0,0 +1,24 @@
+using NexusMods.Abstractions.Settings;
+
+namespace NexusMods.Collections;
+
+public class DownloadSettings : ISettings
+{
+    public int MaxParallelDownloads { get; set; } = Environment.ProcessorCount;
+
+    public static ISettingsBuilder Configure(ISettingsBuilder settingsBuilder)
+    {
+        return settingsBuilder.AddToUI<DownloadSettings>(builder => builder
+            .AddPropertyToUI(x => x.MaxParallelDownloads, propertyBuilder => propertyBuilder
+                .AddToSection(Sections.General)
+                .WithDisplayName("Max Parallel Downloads")
+                .WithDescription("Set the maximum number of downloads that can happen in parallel when downloading collections")
+                .UseSingleValueMultipleChoiceContainer(
+                    valueComparer: EqualityComparer<int>.Default,
+                    allowedValues: Enumerable.Range(start: 1, Environment.ProcessorCount).ToArray(),
+                    valueToDisplayString: static i => i.ToString()
+                )
+            )
+        );
+    }
+}

src/NexusMods.Collections/Services.cs

@@ -2,6 +2,7 @@
 using NexusMods.Abstractions.Collections;
 using NexusMods.Abstractions.Loadouts;
 using NexusMods.Abstractions.NexusModsLibrary;
+using NexusMods.Abstractions.Settings;
 
 namespace NexusMods.Collections;
 
@@ -16,6 +17,7 @@ public static IServiceCollection AddNexusModsCollections(this IServiceCollection
             .AddNexusCollectionItemLoadoutGroupModel()
             .AddNexusCollectionReplicatedLoadoutGroupModel()
             .AddCollectionVerbs()
-            .AddSingleton<CollectionDownloader>();
+            .AddSingleton<CollectionDownloader>()
+            .AddSettings<DownloadSettings>();
     }
 }

src/NexusMods.CrossPlatform/ProtocolRegistration/ProtocolRegistrationLinux.cs

@@ -47,7 +47,10 @@ public ProtocolRegistrationLinux(
     /// <inheritdoc/>
     public async Task RegisterHandler(string uriScheme, bool setAsDefaultHandler = true, CancellationToken cancellationToken = default)
     {
-        if (ApplicationConstants.InstallationMethod != InstallationMethod.PackageManager)
+        var canWriteDesktopFile = ApplicationConstants.InstallationMethod is InstallationMethod.AppImage or InstallationMethod.Manually;
+        var canRegisterAsDefault = ApplicationConstants.InstallationMethod is not InstallationMethod.Flatpak and not InstallationMethod.PackageManager;
+
+        if (canWriteDesktopFile)
         {
             var applicationsDirectory = _fileSystem.GetKnownPath(KnownPath.XDG_DATA_HOME).Combine("applications");
             _logger.LogInformation("Using applications directory `{Path}`", applicationsDirectory);
@@ -73,7 +76,7 @@ public async Task RegisterHandler(string uriScheme, bool setAsDefaultHandler = t
             }
         }
 
-        if (setAsDefaultHandler)
+        if (setAsDefaultHandler && canRegisterAsDefault)
         {
             try
             {