More http download fixes

src/NexusMods.Collections/CollectionDownloader.cs

@@ -15,6 +15,7 @@
 using NexusMods.Abstractions.NexusWebApi;
 using NexusMods.Abstractions.NexusWebApi.Types;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Abstractions.Settings;
 using NexusMods.Abstractions.Telemetry;
 using NexusMods.CrossPlatform.Process;
 using NexusMods.Extensions.BCL;
@@ -256,7 +257,6 @@ public async ValueTask DownloadItems(
         CollectionRevisionMetadata.ReadOnly revisionMetadata,
         ItemType itemType,
         IDb db,
-        int maxDegreeOfParallelism = -1,
         CancellationToken cancellationToken = default)
     {
         var job = new DownloadCollectionJob
@@ -266,7 +266,7 @@ public async ValueTask DownloadItems(
             RevisionMetadata = revisionMetadata,
             Db = db,
             ItemType = itemType,
-            MaxDegreeOfParallelism = maxDegreeOfParallelism,
+            MaxDegreeOfParallelism = _serviceProvider.GetRequiredService<ISettingsManager>().Get<DownloadSettings>().MaxParallelDownloads,
         };
 
         await _jobMonitor.Begin<DownloadCollectionJob, R3.Unit>(job);

src/NexusMods.Collections/DownloadCollectionJob.cs

@@ -12,7 +12,7 @@ public class DownloadCollectionJob : IJobDefinitionWithStart<DownloadCollectionJ
     public required CollectionDownloader.ItemType ItemType { get; init; }
     public required CollectionDownloader Downloader { get; init; }
     public required IDb Db { get; init; }
-    public int MaxDegreeOfParallelism { get; init; } = -1;
+    public required int MaxDegreeOfParallelism { get; init; }
 
     public async ValueTask<R3.Unit> StartAsync(IJobContext<DownloadCollectionJob> context)
     {

src/NexusMods.Collections/DownloadSettings.cs

@@ -0,0 +1,24 @@
+using NexusMods.Abstractions.Settings;
+
+namespace NexusMods.Collections;
+
+public class DownloadSettings : ISettings
+{
+    public int MaxParallelDownloads { get; set; } = Environment.ProcessorCount;
+
+    public static ISettingsBuilder Configure(ISettingsBuilder settingsBuilder)
+    {
+        return settingsBuilder.AddToUI<DownloadSettings>(builder => builder
+            .AddPropertyToUI(x => x.MaxParallelDownloads, propertyBuilder => propertyBuilder
+                .AddToSection(Sections.General)
+                .WithDisplayName("Max Parallel Downloads")
+                .WithDescription("Set the maximum number of downloads that can happen in parallel when downloading collections")
+                .UseSingleValueMultipleChoiceContainer(
+                    valueComparer: EqualityComparer<int>.Default,
+                    allowedValues: Enumerable.Range(start: 1, Environment.ProcessorCount).ToArray(),
+                    valueToDisplayString: static i => i.ToString()
+                )
+            )
+        );
+    }
+}

src/NexusMods.Collections/Services.cs

@@ -2,6 +2,7 @@
 using NexusMods.Abstractions.Collections;
 using NexusMods.Abstractions.Loadouts;
 using NexusMods.Abstractions.NexusModsLibrary;
+using NexusMods.Abstractions.Settings;
 
 namespace NexusMods.Collections;
 
@@ -16,6 +17,7 @@ public static IServiceCollection AddNexusModsCollections(this IServiceCollection
             .AddNexusCollectionItemLoadoutGroupModel()
             .AddNexusCollectionReplicatedLoadoutGroupModel()
             .AddCollectionVerbs()
-            .AddSingleton<CollectionDownloader>();
+            .AddSingleton<CollectionDownloader>()
+            .AddSettings<DownloadSettings>();
     }
 }

src/NexusMods.Networking.HttpDownloader/HttpDownloadJob.cs

@@ -1,18 +1,18 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Collections.Immutable;
 using System.Net;
 using System.Net.Http.Headers;
+using System.Net.Sockets;
 using DynamicData.Kernel;
 using JetBrains.Annotations;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Http.Resilience;
 using Microsoft.Extensions.Logging;
 using NexusMods.Abstractions.HttpDownloads;
 using NexusMods.Abstractions.Jobs;
 using NexusMods.Abstractions.Library.Models;
-using NexusMods.App.BuildInfo;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.Paths;
 using Polly;
+using Polly.Retry;
 
 namespace NexusMods.Networking.HttpDownloader;
 
@@ -22,9 +22,12 @@ namespace NexusMods.Networking.HttpDownloader;
 [PublicAPI]
 public record HttpDownloadJob : IJobDefinitionWithStart<HttpDownloadJob, AbsolutePath>, IHttpDownloadJob
 {
-#pragma warning disable EXTEXP0001
-    private static readonly HttpClient Client = BuildClient();
-#pragma warning restore EXTEXP0001
+    private static readonly ResiliencePipeline<AbsolutePath> ResiliencePipeline = BuildResiliencePipeline();
+
+    /// <summary>
+    /// Client.
+    /// </summary>
+    public required HttpClient Client { get; init; }
 
     /// <summary>
     /// Logger.
@@ -76,15 +79,29 @@ public static IJobTask<HttpDownloadJob, AbsolutePath> Create(
             DownloadPageUri = downloadPage,
             Destination = destination,
             Logger = provider.GetRequiredService<ILogger<HttpDownloadJob>>(),
+            Client = provider.GetRequiredService<HttpClient>(),
         };
 
         return monitor.Begin<HttpDownloadJob, AbsolutePath>(job);
     }
 
-    /// <summary>
-    /// Execute the job
-    /// </summary>
+    /// <inheritdoc/>
     public async ValueTask<AbsolutePath> StartAsync(IJobContext<HttpDownloadJob> context)
+    {
+        var result = await ResiliencePipeline.ExecuteAsync(
+            callback: static (tuple, _) =>
+            {
+                var (self, context) = tuple;
+                return self.StartAsyncImpl(context);
+            },
+            state: (this, context),
+            cancellationToken: context.CancellationToken
+        );
+
+        return result;
+    }
+
+    private async ValueTask<AbsolutePath> StartAsyncImpl(IJobContext<HttpDownloadJob> context)
     {
         await context.YieldAsync();
         await FetchMetadata(context);
@@ -168,6 +185,11 @@ public async ValueTask<AbsolutePath> StartAsync(IJobContext<HttpDownloadJob> con
         {
             await response.Content.CopyToAsync(outputStream, context.CancellationToken);
         }
+        catch (Exception e)
+        {
+            Logger.LogWarning(e, "Exception while downloading from `{PageUri}`, downloaded `{DownloadedBytes}` from `{TotalBytes}` bytes", DownloadPageUri, outputStream.Position, outputStream.Length);
+            throw;
+        }
         finally
         {
             TotalBytesDownloaded = Size.FromLong(outputStream.Position);
@@ -244,36 +266,27 @@ private async ValueTask FetchMetadata(IJobContext context)
         var contentLength = response.Content.Headers.ContentLength;
         ContentLength = contentLength is not null ? Size.FromLong(contentLength.Value) : Optional<Size>.None;
     }
-
-    [Experimental("EXTEXP0001")]
-    private static HttpClient BuildClient()
-    {
-        // TODO: get values from settings, probably make this a singleton
 
-        var pipeline = new ResiliencePipelineBuilder<HttpResponseMessage>()
-            .AddRetry(new HttpRetryStrategyOptions())
-            .Build();
-
-        HttpMessageHandler handler = new ResilienceHandler(pipeline)
-        {
-            InnerHandler = new SocketsHttpHandler
+    private static ResiliencePipeline<AbsolutePath> BuildResiliencePipeline()
+    {
+        ImmutableArray<Type> networkExceptions =
+        [
+            typeof(HttpIOException),
+            typeof(HttpRequestException),
+            typeof(SocketException),
+        ];
+
+        var pipeline = new ResiliencePipelineBuilder<AbsolutePath>()
+            .AddRetry(new RetryStrategyOptions<AbsolutePath>
             {
-                ConnectTimeout = TimeSpan.FromSeconds(30),
-                KeepAlivePingPolicy = HttpKeepAlivePingPolicy.WithActiveRequests,
-                KeepAlivePingDelay = TimeSpan.FromSeconds(5),
-                KeepAlivePingTimeout = TimeSpan.FromSeconds(20),
-            },
-        };
-
-        var client = new HttpClient(handler)
-        {
-            Timeout = TimeSpan.FromSeconds(20),
-            DefaultRequestVersion = HttpVersion.Version11,
-            DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
-        };
-
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(ApplicationConstants.UserAgent);
+                ShouldHandle = args => ValueTask.FromResult(args.Outcome.Exception is not null && networkExceptions.Contains(args.Outcome.Exception.GetType())),
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = true,
+                MaxRetryAttempts = 3,
+                Delay = TimeSpan.FromSeconds(3),
+            })
+            .Build();
 
-        return client;
+        return pipeline;
     }
 }

src/NexusMods.Networking.HttpDownloader/Services.cs

@@ -1,5 +1,8 @@
+using System.Net;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Http.Resilience;
 using NexusMods.App.BuildInfo;
+using Polly;
 
 namespace NexusMods.Networking.HttpDownloader;
 
@@ -12,9 +15,36 @@ public static IServiceCollection AddHttpDownloader(this IServiceCollection servi
     {
         return services.AddSingleton<HttpClient>(_ =>
         {
-            var client = new HttpClient();
-            client.DefaultRequestHeaders.UserAgent.ParseAdd(ApplicationConstants.UserAgent);
+            var client = BuildClient();
             return client;
         });
     }
+
+    private static HttpClient BuildClient()
+    {
+        var pipeline = new ResiliencePipelineBuilder<HttpResponseMessage>()
+            .AddRetry(new HttpRetryStrategyOptions
+            {
+                BackoffType = DelayBackoffType.Exponential,
+                MaxRetryAttempts = 3,
+                Delay = TimeSpan.FromSeconds(3),
+                UseJitter = true,
+            })
+            .Build();
+
+        HttpMessageHandler handler = new ResilienceHandler(pipeline)
+        {
+            InnerHandler = new SocketsHttpHandler(),
+        };
+
+        var client = new HttpClient(handler)
+        {
+            DefaultRequestVersion = HttpVersion.Version11,
+            DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
+        };
+
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(ApplicationConstants.UserAgent);
+
+        return client;
+    }
 }

src/NexusMods.Networking.NexusWebApi/ExternalDownloadJob.cs

@@ -51,6 +51,7 @@ public static IJobTask<ExternalDownloadJob, AbsolutePath> Create(IServiceProvide
             DownloadPageUri = uri,
             Destination = tempFileManager.CreateFile(),
             Uri = uri,
+            Client = provider.GetRequiredService<HttpClient>(),
         };
 
         return monitor.Begin<ExternalDownloadJob, AbsolutePath>(job);