site

Author: KSemenenko

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: build-and-test
 
 on:
   pull_request:

.github/workflows/jekyll-gh-pages.yml

@@ -83,13 +83,24 @@ If no new rule is detected → do not update the file.
 - Docs live in `docs/` and `README.md`
 - Keep a GitHub Pages docs site in sync with `docs/`, using `DOCS-EXAMPLE/` as the reference template for structure and CI/pipeline
 - When adding new docs pages under `docs/Features/`, `docs/ADR/`, or `docs/API/`, also update the corresponding `index.md` to link the page so it’s discoverable in the docs catalog/navigation (the site generator will still publish the page even without the link)
+- Docs site navigation: do not include a `Templates` page (keep templates in-repo, but don’t surface a dedicated docs-site section for them)
+- Docs content: when referencing repo file paths (e.g. `Tests/.../X.cs`), make them clickable by linking to the corresponding GitHub `blob/tree` URL
 - Update docs when behaviour changes
 - Update configuration examples when required
 - Documentation must include clear schemas/diagrams (prefer Mermaid) for every non-trivial feature and integration so GitHub users can understand flows quickly
 - When adding new projects/providers, ensure `README.md` clearly documents installation, DI wiring, and basic usage examples
 - Where feasible, prefer provider options that can build vendor SDK clients from credentials (to reduce consumer boilerplate) while still allowing client injection for advanced scenarios
 - Avoid "ownership flags" like `ownsClient`; prefer a clear swap point (wrapper/factory) so lifetime and disposal rules stay simple and predictable
 - For providers that rely on vendor SDK clients (Graph/Drive/Dropbox/etc.), document how to obtain credentials/keys/tokens and include a minimal code snippet that builds the required SDK client instance
+- CloudKit docs: explicitly clarify that `ContainerId` is a CloudKit container identifier (not a secret) tied to an Apple App ID, and document the optional `HttpClient`/`ICloudKitClient` injection points for advanced HTTP customization and testing
+- Credentials docs: keep provider sections consistent (What you need → Typical steps → Minimal SDK/DI snippet → Suggested configuration keys) so consumers can wire providers without guesswork
+- Docs: keep the testing strategy discoverable from `docs/Development/setup.md` (link or embedded section) so users find how/why tests run during initial setup
+- Docs: validate all Mermaid diagrams render on the docs site (Mermaid v10.9.5) and fix any syntax errors before shipping docs changes
+- Docs site: include `sitemap.xml` and reference it from `robots.txt` so search engines can discover all pages after every rebuild
+- Docs site: display the project version from `Directory.Build.props` (not CI run numbers) and keep the footer copyright line slightly smaller than the rest for a cleaner visual hierarchy
+- Docs site: display the short project name `Storage` in the site title/nav (keep `ManagedCode.Storage` in content where it refers to package IDs)
+- Docs: do not add ADRs for docs-site generation/pipeline changes; document the docs-site build, SEO, and GitHub Pages workflow under `docs/Development/` instead
+- Docs site: do not generate redirect/alias pages like `/Storage/`; keep a single canonical home URL (`/`) and remove unused routes
 
 ### Testing (ALL TASKS)
 
@@ -143,6 +154,7 @@ If no new rule is detected → do not update the file.
 - Suffix async APIs with `Async`; keep test names aligned with existing patterns (e.g., `DownloadFile_WhenFileExists_ReturnsSuccess`)
 - Remove unused usings and let analyzers guide layout
 - When a `foreach` loop’s first step is just transforming the iteration variable (e.g., `var y = Map(x)`), prefer mapping the sequence explicitly with `.Select(...)` so intent is clearer and analyzers stay quiet
+- Avoid buffering whole files into `MemoryStream` in product code (assume multi‑GB files); stream directly to the destination (response stream / file stream) and use incremental hashing/CRC when you need verification
 - No magic literals — extract to constants, enums, or config when it improves clarity
 
 ### Git & PRs

AGENTS.md

@@ -146,15 +146,13 @@ public virtual async Task<ActionResult> DownloadBytesAsync([FromRoute] string pa
             return Problem("File name is required", statusCode: StatusCodes.Status400BadRequest);
         }
 
-        var download = await Storage.DownloadAsync(path, cancellationToken);
-        if (download.IsFailed)
+        var streamResult = await Storage.GetStreamAsync(path, cancellationToken);
+        if (streamResult.IsFailed)
         {
-            return Problem(download.Problem?.Title ?? "File not found", statusCode: (int?)download.Problem?.StatusCode ?? StatusCodes.Status404NotFound);
+            return Problem(streamResult.Problem?.Title ?? "File not found", statusCode: (int?)streamResult.Problem?.StatusCode ?? StatusCodes.Status404NotFound);
         }
 
-        await using var tempStream = new MemoryStream();
-        await download.Value.FileStream.CopyToAsync(tempStream, cancellationToken);
-        return File(tempStream.ToArray(), MimeHelper.GetMimeType(path), path);
+        return File(streamResult.Value, MimeHelper.GetMimeType(path), path, enableRangeProcessing: _options.EnableRangeProcessing);
     }
 
     /// <inheritdoc />
@@ -237,6 +235,11 @@ public class StorageServerOptions
     /// </summary>
     public const int DefaultInMemoryUploadThresholdBytes = 256 * 1024;
 
+    /// <summary>
+    /// Default threshold in bytes for endpoints that return an in-memory payload (for example: <see cref="FileContentResult"/>).
+    /// </summary>
+    public const int DefaultInMemoryDownloadThresholdBytes = 256 * 1024;
+
     /// <summary>
     /// Default boundary length limit applied to multipart requests.
     /// </summary>
@@ -262,6 +265,12 @@ public class StorageServerOptions
     /// </summary>
     public int InMemoryUploadThresholdBytes { get; set; } = DefaultInMemoryUploadThresholdBytes;
 
+    /// <summary>
+    /// Gets or sets the maximum payload size (in bytes) that will be materialised in memory when an API returns the entire payload as bytes.
+    /// Use streaming endpoints for larger payloads.
+    /// </summary>
+    public int InMemoryDownloadThresholdBytes { get; set; } = DefaultInMemoryDownloadThresholdBytes;
+
     /// <summary>
     /// Gets or sets the maximum allowed length for multipart boundaries when parsing raw upload streams.
     /// </summary>

Integraions/ManagedCode.Storage.Server/Controllers/StorageControllerBase.cs

@@ -1,10 +1,13 @@
+using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using ManagedCode.MimeTypes;
 using ManagedCode.Storage.Core;
+using ManagedCode.Storage.Server.Controllers;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
 using IResult = Microsoft.AspNetCore.Http.IResult;
 
 namespace ManagedCode.Storage.Server.Extensions.Controller;
@@ -14,6 +17,12 @@ namespace ManagedCode.Storage.Server.Extensions.Controller;
 /// </summary>
 public static class ControllerDownloadExtensions
 {
+    private static StorageServerOptions ResolveServerOptions(ControllerBase controller)
+    {
+        var services = controller.HttpContext?.RequestServices;
+        return services?.GetService<StorageServerOptions>() ?? new StorageServerOptions();
+    }
+
     /// <summary>
     /// Streams the specified blob to the caller using <see cref="IResult"/>.
     /// </summary>
@@ -61,13 +70,24 @@ public static async Task<FileContentResult> DownloadAsFileContentResultAsync(
         string blobName,
         CancellationToken cancellationToken = default)
     {
+        var serverOptions = ResolveServerOptions(controller);
+
         var result = await storage.DownloadAsync(blobName, cancellationToken);
         if (result.IsFailed)
             throw new FileNotFoundException(blobName);
 
-        using var memoryStream = new MemoryStream();
-        await result.Value.FileStream.CopyToAsync(memoryStream, cancellationToken);
-        return new FileContentResult(memoryStream.ToArray(), MimeHelper.GetMimeType(blobName))
+        await using var localFile = result.Value;
+
+        var length = localFile.FileInfo.Length;
+        if (length > serverOptions.InMemoryDownloadThresholdBytes)
+        {
+            throw new InvalidOperationException(
+                $"Blob '{blobName}' is {length} bytes which exceeds the in-memory download threshold of {serverOptions.InMemoryDownloadThresholdBytes} bytes. " +
+                "Use DownloadAsFileResultAsync or DownloadAsStreamAsync for large files.");
+        }
+
+        var bytes = await localFile.ReadAllBytesAsync(cancellationToken);
+        return new FileContentResult(bytes, MimeHelper.GetMimeType(blobName))
         {
             FileDownloadName = blobName
         };

Integraions/ManagedCode.Storage.Server/Extensions/Controller/ControllerDownloadExtensions.cs

@@ -45,7 +45,7 @@ public static async Task<BlobMetadata> UploadFormFileAsync(
         var serverOptions = ResolveServerOptions(controller);
         if (file.Length > serverOptions.InMemoryUploadThresholdBytes)
         {
-            var localFile = await file.ToLocalFileAsync(cancellationToken);
+            await using var localFile = await file.ToLocalFileAsync(cancellationToken);
             var result = await storage.UploadAsync(localFile.FileInfo, uploadOptions, cancellationToken);
             result.ThrowIfFail();
             return result.Value!;
@@ -73,7 +73,7 @@ public static async Task<BlobMetadata> UploadFromBrowserFileAsync(
 
         if (file.Size > serverOptions.InMemoryUploadThresholdBytes)
         {
-            var localFile = await file.ToLocalFileAsync(cancellationToken);
+            await using var localFile = await file.ToLocalFileAsync(cancellationToken);
             var result = await storage.UploadAsync(localFile.FileInfo, uploadOptions, cancellationToken);
             result.ThrowIfFail();
             return result.Value!;
@@ -155,11 +155,7 @@ public static async Task<BlobMetadata> UploadFromStreamAsync(
 
                 uploadOptions ??= new UploadOptions(fileName, mimeType: contentType);
 
-                using var memoryStream = new MemoryStream();
-                await section.Body.CopyToAsync(memoryStream, cancellationToken);
-                memoryStream.Position = 0;
-
-                var result = await storage.UploadAsync(memoryStream, uploadOptions, cancellationToken);
+                var result = await storage.UploadAsync(section.Body, uploadOptions, cancellationToken);
                 result.ThrowIfFail();
                 return result.Value!;
             }

Integraions/ManagedCode.Storage.Server/Extensions/Controller/ControllerUploadExtensions.cs

@@ -10,7 +10,10 @@ public static class BrowserFileExtensions
     public static async Task<LocalFile> ToLocalFileAsync(this IBrowserFile formFile, CancellationToken cancellationToken = default)
     {
         var localFile = LocalFile.FromRandomNameWithExtension(formFile.Name);
-        await localFile.CopyFromStreamAsync(formFile.OpenReadStream(), cancellationToken);
+        await using (var stream = formFile.OpenReadStream())
+        {
+            await localFile.CopyFromStreamAsync(stream, cancellationToken);
+        }
         return localFile;
     }
 }

Integraions/ManagedCode.Storage.Server/Extensions/File/BrowserFileExtensions.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -12,14 +13,19 @@ public static class FormFileExtensions
     public static async Task<LocalFile> ToLocalFileAsync(this IFormFile formFile, CancellationToken cancellationToken = default)
     {
         var localFile = LocalFile.FromRandomNameWithExtension(formFile.FileName);
-        await localFile.CopyFromStreamAsync(formFile.OpenReadStream(), cancellationToken);
+        await using (var stream = formFile.OpenReadStream())
+        {
+            await localFile.CopyFromStreamAsync(stream, cancellationToken);
+        }
         return localFile;
     }
 
     public static async IAsyncEnumerable<LocalFile> ToLocalFilesAsync(this IFormFileCollection formFileCollection,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        foreach (var formFile in formFileCollection)
-            yield return await formFile.ToLocalFileAsync(cancellationToken);
+        foreach (var localFileTask in formFileCollection.Select(formFile => formFile.ToLocalFileAsync(cancellationToken)))
+        {
+            yield return await localFileTask;
+        }
     }
 }

Integraions/ManagedCode.Storage.Server/Extensions/File/FormFileExtensions.cs

@@ -184,8 +184,7 @@ public async Task<Stream> OpenWriteAsync(
             }
         }
 
-        // For now, return a memory stream that will be uploaded when disposed
-        // This is a simplified implementation - real streaming would require provider-specific support
+        // Return a file-backed write stream that uploads on dispose to avoid buffering large payloads in memory.
         return new VfsWriteStream(_vfs.Storage, _path.ToBlobKey(), options, _cache, _vfs.Options, _logger);
     }
 
@@ -217,10 +216,22 @@ public async Task<byte[]> ReadAllBytesAsync(CancellationToken cancellationToken
     {
         _logger.LogDebug("Reading all bytes: {Path}", _path);
 
-        await using var stream = await OpenReadAsync(cancellationToken: cancellationToken);
-        using var memoryStream = new MemoryStream();
-        await stream.CopyToAsync(memoryStream, cancellationToken);
-        return memoryStream.ToArray();
+        await EnsureMetadataLoadedAsync(cancellationToken);
+
+        if (_blobMetadata == null)
+        {
+            throw new VfsNotFoundException(_path);
+        }
+
+        var result = await _vfs.Storage.DownloadAsync(_path.ToBlobKey(), cancellationToken);
+
+        if (!result.IsSuccess || result.Value == null)
+        {
+            throw new VfsOperationException($"Failed to read all bytes for file: {_path}");
+        }
+
+        await using var localFile = result.Value;
+        return await localFile.ReadAllBytesAsync(cancellationToken);
     }
 
     /// <inheritdoc />
@@ -269,7 +280,9 @@ public async ValueTask<IReadOnlyDictionary<string, string>> GetMetadataAsync(
     {
         var cacheKey = $"file_custom_metadata:{_vfs.ContainerName}:{_path}";
 
-        if (_vfs.Options.EnableCache && _cache.TryGetValue(cacheKey, out IReadOnlyDictionary<string, string> cached))
+        if (_vfs.Options.EnableCache
+            && _cache.TryGetValue(cacheKey, out IReadOnlyDictionary<string, string>? cached)
+            && cached is not null)
         {
             _logger.LogDebug("File metadata (cached): {Path}", _path);
             return cached;
@@ -281,7 +294,7 @@ public async ValueTask<IReadOnlyDictionary<string, string>> GetMetadataAsync(
         {
             _cache.Set(cacheKey, metadata, _vfs.Options.CacheTTL);
             var metadataKey = $"file_metadata:{_vfs.ContainerName}:{_path}";
-            if (_cache.TryGetValue(metadataKey, out MetadataCacheEntry entry))
+            if (_cache.TryGetValue(metadataKey, out MetadataCacheEntry? entry) && entry is not null)
             {
                 entry.CustomMetadata = metadata;
                 _cache.Set(metadataKey, entry, _vfs.Options.CacheTTL);
@@ -358,7 +371,7 @@ private async Task EnsureMetadataLoadedAsync(CancellationToken cancellationToken
         if (_vfs.Options.EnableCache)
         {
             var metadataKey = $"file_metadata:{_vfs.ContainerName}:{_path}";
-            if (_cache.TryGetValue(metadataKey, out MetadataCacheEntry entry))
+            if (_cache.TryGetValue(metadataKey, out MetadataCacheEntry? entry) && entry is not null)
             {
                 _vfsMetadata = entry.Metadata;
                 _blobMetadata = entry.BlobMetadata;
@@ -376,4 +389,4 @@ private async Task EnsureMetadataLoadedAsync(CancellationToken cancellationToken
         // This is a simplified ETag extraction - real implementation would depend on the storage provider
         return null;
     }
-}
+}