refactroring + tests

Author: KSemenenko

AGENTS.md

@@ -13,6 +13,8 @@ If I tell you to remember something, you do the same, update
 - Deliver ASP.NET integrations that expose upload/download controllers, SignalR streaming, and matching HTTP and SignalR clients built on the storage layer for files, streams, and chunked transfers.
 - Provide base ASP.NET controllers with minimal routing so consumers can inherit and customize routes, authorization, and behaviors without rigid defaults.
 - Favor controller extension patterns and optionally expose interfaces to guide consumers on recommended actions so they can implement custom endpoints easily.
+- For comprehensive storage platform upgrades, follow the nine-step flow: solidify SignalR streaming hub/client with logging and tests, harden controller upload paths (standard/stream/chunked) with large-file coverage, add keyed DI registrations and cross-provider sync fixtures, extend VFS with keyed support and >1 GB trials, create streamed large-file/CRC helpers, run end-to-end suites (controllers, SignalR, VFS, cross-provider), verify Blazor upload extensions, expand docs with VFS + provider identity guidance + keyed samples, and finish by running the full preview-enabled test suite addressing warnings.
+- Normalise MIME lookups through `MimeHelper`; avoid ad-hoc MIME resolution helpers so all content-type logic flows through its APIs.
 
 # Repository Guidelines
 

Integraions/ManagedCode.Storage.Client.SignalR/ManagedCode.Storage.Client.SignalR.csproj

@@ -22,8 +22,8 @@
 
     <ItemGroup>
         <PackageReference Include="ManagedCode.MimeTypes" Version="1.0.3" />
-        <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
-        <PackageReference Include="Microsoft.AspNetCore.Http.Connections.Client" Version="8.0.8" />
+        <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.9" />
+        <PackageReference Include="Microsoft.AspNetCore.Http.Connections.Client" Version="9.0.9" />
     </ItemGroup>
 
     <ItemGroup>

Integraions/ManagedCode.Storage.Client.SignalR/StorageSignalRClient.cs

@@ -3,8 +3,8 @@
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Runtime.CompilerServices;
 using System.Threading;
-using System.Threading.Channels;
 using System.Threading.Tasks;
 using ManagedCode.Storage.Client.SignalR.Models;
 using Microsoft.AspNetCore.SignalR;
@@ -158,17 +158,11 @@ public async Task<StorageTransferStatus> UploadAsync(Stream stream, StorageUploa
             throw new ArgumentException("The upload descriptor must contain a file name.", nameof(descriptor));
         }
 
-        descriptor.TransferId = string.IsNullOrWhiteSpace(descriptor.TransferId)
-            ? Guid.NewGuid().ToString("N")
-            : descriptor.TransferId;
-
         if (stream.CanSeek)
         {
             stream.Seek(0, SeekOrigin.Begin);
         }
 
-        var handler = CreateProgressRelay(descriptor.TransferId, progress);
-
         var bufferSize = _options?.StreamBufferSize ?? 64 * 1024;
         if (bufferSize <= 0)
         {
@@ -181,57 +175,32 @@ public async Task<StorageTransferStatus> UploadAsync(Stream stream, StorageUploa
             throw new InvalidOperationException("UploadChannelCapacity must be greater than zero.");
         }
 
-        var channel = Channel.CreateBounded<byte[]>(new BoundedChannelOptions(channelCapacity)
-        {
-            SingleReader = true,
-            SingleWriter = true,
-            FullMode = BoundedChannelFullMode.Wait
-        });
+        var transferId = await connection.InvokeAsync<string>("BeginUploadStreamAsync", descriptor, cancellationToken).ConfigureAwait(false);
+        descriptor.TransferId = transferId;
 
-        Task producerTask = ProduceChunksAsync(stream, channel.Writer, bufferSize, cancellationToken);
-        var completionSource = new TaskCompletionSource<StorageTransferStatus>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var handler = CreateProgressRelay(transferId, progress);
 
-        void OnCompleted(object? _, StorageTransferStatus status)
-        {
-            if (string.Equals(status.TransferId, descriptor.TransferId, StringComparison.OrdinalIgnoreCase))
-            {
-                completionSource.TrySetResult(status);
-            }
-        }
+        var statusStream = connection.StreamAsync<StorageTransferStatus>(
+            "UploadStreamContentAsync",
+            transferId,
+            ReadChunksAsync(stream, bufferSize, cancellationToken),
+            cancellationToken);
 
-        void OnFaulted(object? _, StorageTransferStatus status)
-        {
-            if (string.Equals(status.TransferId, descriptor.TransferId, StringComparison.OrdinalIgnoreCase))
-            {
-                completionSource.TrySetException(new HubException(status.Error ?? "Upload failed."));
-            }
-        }
+        StorageTransferStatus? lastStatus = null;
 
-        void OnCanceled(object? _, StorageTransferStatus status)
+        try
         {
-            if (string.Equals(status.TransferId, descriptor.TransferId, StringComparison.OrdinalIgnoreCase))
+            await foreach (var status in statusStream.WithCancellation(cancellationToken).ConfigureAwait(false))
             {
-                completionSource.TrySetCanceled(cancellationToken);
+                lastStatus = status;
             }
         }
-
-        TransferCompleted += OnCompleted;
-        TransferFaulted += OnFaulted;
-        TransferCanceled += OnCanceled;
-
-        try
-        {
-            await connection.SendAsync("UploadStreamAsync", descriptor, channel.Reader, cancellationToken).ConfigureAwait(false);
-            await producerTask.ConfigureAwait(false);
-            return await completionSource.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
-        }
         finally
         {
-            TransferCompleted -= OnCompleted;
-            TransferFaulted -= OnFaulted;
-            TransferCanceled -= OnCanceled;
             handler?.Dispose();
         }
+
+        return lastStatus ?? throw new HubException($"Upload stream for transfer '{transferId}' completed without status.");
     }
 
     /// <inheritdoc />
@@ -261,14 +230,35 @@ public async Task<StorageTransferStatus> DownloadAsync(string blobName, Stream d
 
         destination.Flush();
 
-        return lastStatus ?? new StorageTransferStatus
+        if (lastStatus is null)
         {
-            Operation = "download",
-            ResourceName = blobName,
-            BytesTransferred = totalBytes,
-            TotalBytes = totalBytes,
-            IsCompleted = true
-        };
+            return new StorageTransferStatus
+            {
+                Operation = "download",
+                ResourceName = blobName,
+                BytesTransferred = totalBytes,
+                TotalBytes = totalBytes,
+                IsCompleted = true
+            };
+        }
+
+        if (!lastStatus.IsCompleted)
+        {
+            lastStatus = new StorageTransferStatus
+            {
+                TransferId = lastStatus.TransferId,
+                Operation = lastStatus.Operation,
+                ResourceName = lastStatus.ResourceName,
+                BytesTransferred = lastStatus.BytesTransferred > 0 ? lastStatus.BytesTransferred : totalBytes,
+                TotalBytes = lastStatus.TotalBytes ?? totalBytes,
+                IsCompleted = true,
+                IsCanceled = lastStatus.IsCanceled,
+                Error = lastStatus.Error,
+                Metadata = lastStatus.Metadata
+            };
+        }
+
+        return lastStatus;
     }
 
     /// <inheritdoc />
@@ -444,27 +434,22 @@ private IDisposable CreateDownloadProgressRelay(string blobName, IProgress<Stora
         });
     }
 
-    private static async Task ProduceChunksAsync(Stream source, ChannelWriter<byte[]> writer, int bufferSize, CancellationToken cancellationToken)
+    private static async IAsyncEnumerable<byte[]> ReadChunksAsync(
+        Stream source,
+        int bufferSize,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var buffer = new byte[bufferSize];
-
-        try
+        while (true)
         {
-            while (true)
+            int read = await source.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false);
+            if (read <= 0)
             {
-                var read = await source.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false);
-                if (read <= 0)
-                {
-                    break;
-                }
-
-                var chunk = buffer.Take(read).ToArray();
-                await writer.WriteAsync(chunk, cancellationToken).ConfigureAwait(false);
+                yield break;
             }
-        }
-        finally
-        {
-            writer.TryComplete();
+
+            var chunk = buffer.AsSpan(0, read).ToArray();
+            yield return chunk;
         }
     }
 

Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs

@@ -218,8 +218,28 @@ public static class StorageServerHeaders
 /// </summary>
 public class StorageServerOptions
 {
+    /// <summary>
+    /// Default threshold in bytes after which uploads are buffered to disk instead of kept in memory.
+    /// </summary>
+    public const int DefaultInMemoryUploadThresholdBytes = 256 * 1024;
+
+    /// <summary>
+    /// Default boundary length limit applied to multipart requests.
+    /// </summary>
+    public const int DefaultMultipartBoundaryLengthLimit = 70;
+
     /// <summary>
     /// Gets or sets a value indicating whether range processing is enabled for streaming responses.
     /// </summary>
     public bool EnableRangeProcessing { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets the maximum payload size (in bytes) that will be buffered in memory before switching to a file-backed upload path.
+    /// </summary>
+    public int InMemoryUploadThresholdBytes { get; set; } = DefaultInMemoryUploadThresholdBytes;
+
+    /// <summary>
+    /// Gets or sets the maximum allowed length for multipart boundaries when parsing raw upload streams.
+    /// </summary>
+    public int MultipartBoundaryLengthLimit { get; set; } = DefaultMultipartBoundaryLengthLimit;
 }

Integraions/ManagedCode.Storage.Server/Extensions/Controller/ControllerUploadExtensions.cs

@@ -2,9 +2,10 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using ManagedCode.Communication;
 using ManagedCode.Storage.Core;
 using ManagedCode.Storage.Core.Models;
-using ManagedCode.Communication;
+using ManagedCode.Storage.Server.Controllers;
 using ManagedCode.Storage.Server.ChunkUpload;
 using ManagedCode.Storage.Server.Extensions.File;
 using ManagedCode.Storage.Server.Helpers;
@@ -14,6 +15,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.WebUtilities;
 using Microsoft.Net.Http.Headers;
+using Microsoft.Extensions.DependencyInjection;
 
 namespace ManagedCode.Storage.Server.Extensions.Controller;
 
@@ -22,8 +24,11 @@ namespace ManagedCode.Storage.Server.Extensions.Controller;
 /// </summary>
 public static class ControllerUploadExtensions
 {
-    private const int DefaultMultipartBoundaryLengthLimit = 70;
-    private const int MinLengthForLargeFile = 256 * 1024;
+    private static StorageServerOptions ResolveServerOptions(ControllerBase controller)
+    {
+        var services = controller.HttpContext?.RequestServices;
+        return services?.GetService<StorageServerOptions>() ?? new StorageServerOptions();
+    }
 
    /// <summary>
    /// Uploads a form file to storage and returns blob metadata.
@@ -32,21 +37,22 @@ public static async Task<BlobMetadata> UploadFormFileAsync(
     this ControllerBase controller,
     IStorage storage,
     IFormFile file,
-    UploadOptions? options = null,
+    UploadOptions? uploadOptions = null,
     CancellationToken cancellationToken = default)
 {
-    options ??= new UploadOptions(file.FileName, mimeType: file.ContentType);
+    uploadOptions ??= new UploadOptions(file.FileName, mimeType: file.ContentType);
 
-        if (file.Length > MinLengthForLargeFile)
+        var serverOptions = ResolveServerOptions(controller);
+        if (file.Length > serverOptions.InMemoryUploadThresholdBytes)
         {
             var localFile = await file.ToLocalFileAsync(cancellationToken);
-            var result = await storage.UploadAsync(localFile.FileInfo, options, cancellationToken);
+            var result = await storage.UploadAsync(localFile.FileInfo, uploadOptions, cancellationToken);
             result.ThrowIfFail();
             return result.Value!;
         }
 
         await using var stream = file.OpenReadStream();
-        var uploadResult = await storage.UploadAsync(stream, options, cancellationToken);
+        var uploadResult = await storage.UploadAsync(stream, uploadOptions, cancellationToken);
         uploadResult.ThrowIfFail();
         return uploadResult.Value!;
 }
@@ -58,21 +64,23 @@ public static async Task<BlobMetadata> UploadFromBrowserFileAsync(
     this ControllerBase controller,
     IStorage storage,
     IBrowserFile file,
-    UploadOptions? options = null,
+    UploadOptions? uploadOptions = null,
     CancellationToken cancellationToken = default)
 {
-    options ??= new UploadOptions(file.Name, mimeType: file.ContentType);
+    uploadOptions ??= new UploadOptions(file.Name, mimeType: file.ContentType);
+
+    var serverOptions = ResolveServerOptions(controller);
 
-    if (file.Size > MinLengthForLargeFile)
+    if (file.Size > serverOptions.InMemoryUploadThresholdBytes)
     {
         var localFile = await file.ToLocalFileAsync(cancellationToken);
-        var result = await storage.UploadAsync(localFile.FileInfo, options, cancellationToken);
+        var result = await storage.UploadAsync(localFile.FileInfo, uploadOptions, cancellationToken);
         result.ThrowIfFail();
         return result.Value!;
     }
 
     await using var stream = file.OpenReadStream();
-    var uploadResult =  await storage.UploadAsync(stream, options, cancellationToken);
+    var uploadResult =  await storage.UploadAsync(stream, uploadOptions, cancellationToken);
     uploadResult.ThrowIfFail();
     return uploadResult.Value!;
 }
@@ -120,17 +128,19 @@ public static async Task<BlobMetadata> UploadFromStreamAsync(
     this ControllerBase controller,
     IStorage storage,
     HttpRequest request,
-    UploadOptions? options = null,
+    UploadOptions? uploadOptions = null,
     CancellationToken cancellationToken = default)
 {
     if (!StreamHelper.IsMultipartContentType(request.ContentType))
     {
         throw new InvalidOperationException("Not a multipart request");
     }
 
+    var serverOptions = ResolveServerOptions(controller);
+
     var boundary = StreamHelper.GetBoundary(
         MediaTypeHeaderValue.Parse(request.ContentType),
-        DefaultMultipartBoundaryLengthLimit);
+        serverOptions.MultipartBoundaryLengthLimit);
 
     var multipartReader = new MultipartReader(boundary, request.Body);
     var section = await multipartReader.ReadNextSectionAsync(cancellationToken);
@@ -143,13 +153,13 @@ public static async Task<BlobMetadata> UploadFromStreamAsync(
             var fileName = contentDisposition.FileName.Value;
             var contentType = section.ContentType;
 
-            options ??= new UploadOptions(fileName, mimeType: contentType);
+            uploadOptions ??= new UploadOptions(fileName, mimeType: contentType);
 
             using var memoryStream = new MemoryStream();
             await section.Body.CopyToAsync(memoryStream, cancellationToken);
             memoryStream.Position = 0;
 
-            var result = await storage.UploadAsync(memoryStream, options, cancellationToken);
+            var result = await storage.UploadAsync(memoryStream, uploadOptions, cancellationToken);
             result.ThrowIfFail();
             return result.Value!;
         }