Do not buffer @FileResult data into memory

Author: C0nquistadore

src/Dibix.Dapper/DapperDatabaseAccessor.cs

@@ -83,6 +83,8 @@ protected override async Task<IMultipleResultReader> QueryMultipleAsync(string c
             return new DapperGridResultReader(reader, commandText, commandType, parameters, DbProviderAdapter);
         }
 
+        protected override IEnumerable<TReturn> Parse<TReturn>(IDataReader reader) => reader.Parse<TReturn>();
+
         protected override void DisposeConnection()
         {
             if (_onDispose != null)
@@ -97,59 +99,17 @@ protected object CollectParameters(ParametersVisitor parametersVisitor)
         {
             Guard.IsNotNull(parametersVisitor, nameof(parametersVisitor));
             DynamicParameters @params = new DynamicParameters();
-            parametersVisitor.VisitInputParameters((name, dataType, value, size, isOutput, customInputType) =>
-            {
-                object normalizedValue = NormalizeParameterValue(value);
-                DbType? dbType = NormalizeParameterDbType(dataType, customInputType);
-                ParameterDirection? direction = isOutput ? ParameterDirection.Output : null;
-                int? normalizedSize = NormalizeParameterSize(size, dbType, isOutput);
-                @params.Add(name, value: normalizedValue, dbType, direction, normalizedSize);
-            });
+            DapperParameterCollector parameterCollector = new DapperParameterCollector(@params, DbProviderAdapter);
+            parametersVisitor.VisitInputParameters(parameterCollector.VisitInputParameter);
             return new DynamicParametersWrapper(@params, parametersVisitor);
         }
         #endregion
 
         #region Private Methods
-        private object NormalizeParameterValue(object value)
-        {
-            if (value is StructuredType udt)
-                return new DapperStructuredTypeParameter(udt, DbProviderAdapter);
-
-            return value;
-        }
-
-        private static DbType? NormalizeParameterDbType(DbType dbType, CustomInputType customInputType)
-        {
-            if (dbType == DbType.Xml)
-                return null; // You would guess DbType.Xml, but since Dapper treats .NET XML types (i.E. XElement) as custom types, DbType = null is expected
-
-            if (customInputType != default)
-                return null; // Same weird logic like above. Dapper will only resolve the custom type handler, if the db type is null.
-
-            return dbType;
-        }
-
-        private static int? NormalizeParameterSize(int? size, DbType? dbType, bool isOutput)
-        {
-            if (size.HasValue)
-                return size;
-
-            switch (dbType)
-            {
-                case DbType.String when isOutput:
-                case DbType.StringFixedLength when isOutput:
-                case DbType.AnsiString when isOutput:
-                case DbType.AnsiStringFixedLength when isOutput:
-                    return -1;
-
-                default:
-                    return null;
-            }
-        }
-
         private static void ConfigureDapper()
         {
             SqlMapper.AddTypeHandler(new DapperUriTypeHandler());
+            SqlMapper.SetTypeMap(FileEntityTypeMap.Type, new FileEntityTypeMap());
         }
         #endregion
 
@@ -181,6 +141,45 @@ void SqlMapper.IParameterCallbacks.OnCompleted()
                 _parameterCallbacks.OnCompleted();
             }
         }
+
+        private sealed class DapperParameterCollector : DbParameterCollector
+        {
+            private readonly DynamicParameters _dynamicParameters;
+
+            public DapperParameterCollector(DynamicParameters dynamicParameters, DbProviderAdapter dbProviderAdapter) : base(dbProviderAdapter)
+            {
+                _dynamicParameters = dynamicParameters;
+            }
+
+            public override void VisitInputParameter(string name, DbType dataType, object value, int? size, bool isOutput, CustomInputType customInputType)
+            {
+                object normalizedValue = NormalizeParameterValue(value);
+                DbType? dbType = NormalizeParameterDbType(dataType, customInputType);
+                ParameterDirection? direction = isOutput ? ParameterDirection.Output : null;
+                int? normalizedSize = NormalizeParameterSize(size, dbType, isOutput);
+                _dynamicParameters.Add(name, value: normalizedValue, dbType, direction, normalizedSize);
+            }
+
+            private object NormalizeParameterValue(object value)
+            {
+                if (value is StructuredType udt)
+                    return new DapperStructuredTypeParameter(udt, DbProviderAdapter);
+
+                return value;
+            }
+
+            private static DbType? NormalizeParameterDbType(DbType dbType, CustomInputType customInputType)
+            {
+                if (dbType == DbType.Xml)
+                    return null; // You would guess DbType.Xml, but since Dapper treats .NET XML types (i.E. XElement) as custom types, DbType = null is expected
+
+                if (customInputType != default)
+                    return null; // Same weird logic like above. Dapper will only resolve the custom type handler, if the db type is null.
+
+                return dbType;
+            }
+
+        }
         #endregion
     }
 }

src/Dibix.Dapper/DecoratedTypeMap.cs

@@ -4,18 +4,18 @@
 
 namespace Dibix.Dapper
 {
-    internal sealed class DecoratedTypeMap : SqlMapper.ITypeMap
+    internal class DecoratedTypeMap : SqlMapper.ITypeMap
     {
         #region Fields
         private readonly SqlMapper.ITypeMap _inner;
         private readonly Type _type;
         #endregion
 
         #region Constructor
-        private DecoratedTypeMap(SqlMapper.ITypeMap inner, Type type)
+        private protected DecoratedTypeMap(SqlMapper.ITypeMap inner, Type type)
         {
-            this._inner = inner;
-            this._type = type;
+            _inner = inner;
+            _type = type;
         }
         #endregion
 
@@ -46,20 +46,20 @@ private static void Register(Type type)
         #endregion
 
         #region SqlMapper.ITypeMap Members
-        ConstructorInfo SqlMapper.ITypeMap.FindConstructor(string[] names, Type[] types) => this._inner.FindConstructor(names, types);
+        ConstructorInfo SqlMapper.ITypeMap.FindConstructor(string[] names, Type[] types) => _inner.FindConstructor(names, types);
 
-        ConstructorInfo SqlMapper.ITypeMap.FindExplicitConstructor() => this._inner.FindExplicitConstructor();
+        ConstructorInfo SqlMapper.ITypeMap.FindExplicitConstructor() => _inner.FindExplicitConstructor();
 
-        SqlMapper.IMemberMap SqlMapper.ITypeMap.GetConstructorParameter(ConstructorInfo constructor, string columnName) => this._inner.GetConstructorParameter(constructor, columnName);
+        SqlMapper.IMemberMap SqlMapper.ITypeMap.GetConstructorParameter(ConstructorInfo constructor, string columnName) => _inner.GetConstructorParameter(constructor, columnName);
 
-        SqlMapper.IMemberMap SqlMapper.ITypeMap.GetMember(string columnName)
+        public virtual SqlMapper.IMemberMap GetMember(string columnName)
         {
             if (String.IsNullOrEmpty(columnName))
-                throw new InvalidOperationException($"Column name was not specified, therefore it cannot be mapped to type '{this._type}'");
+                throw new InvalidOperationException($"Column name was not specified, therefore it cannot be mapped to type '{_type}'");
 
-            SqlMapper.IMemberMap member = this._inner.GetMember(columnName);
+            SqlMapper.IMemberMap member = _inner.GetMember(columnName);
             if (member == null)
-                throw new InvalidOperationException($"Column '{columnName}' does not match a property on type '{this._type}'");
+                throw new InvalidOperationException($"Column '{columnName}' does not match a property on type '{_type}'");
 
             return member;
         }

src/Dibix.Dapper/FileEntityTypeMap.cs

@@ -0,0 +1,23 @@
+using System;
+using Dapper;
+
+namespace Dibix.Dapper
+{
+    internal sealed class FileEntityTypeMap : DecoratedTypeMap, SqlMapper.ITypeMap
+    {
+        public static readonly Type Type = typeof(FileEntity);
+
+        internal FileEntityTypeMap() : base(SqlMapper.GetTypeMap(Type), Type) { }
+
+        public override SqlMapper.IMemberMap GetMember(string columnName)
+        {
+            // Dapper currently doesn't support reading individual columns as a stream instead of buffering them into memory.
+            // See: https://github.com/DapperLib/Dapper/issues/893
+            // Therefore we skip the data property during the initial read/parse and read it manually using DbDataReader.GetStream().
+            if (String.Equals(columnName, nameof(FileEntity.Data), StringComparison.OrdinalIgnoreCase))
+                return null;
+
+            return base.GetMember(columnName);
+        }
+    }
+}

src/Dibix.Http.Host/Runtime/HttpResponseFormatter.cs

@@ -1,10 +1,10 @@
 using System;
-using System.IO;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Dibix.Http.Server;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.AspNetCore.Http.Headers;
 using Microsoft.Net.Http.Headers;
 
@@ -56,8 +56,18 @@ void AppendFileName(ResponseHeaders responseHeaders, string fileName)
                     responseHeaders.ContentType = new MediaTypeHeaderValue(mediaType);
                     AppendFileName(responseHeaders, fileEntity.FileName);
 
-                    using MemoryStream memoryStream = new MemoryStream(fileEntity.Data);
-                    await memoryStream.CopyToAsync(_response.Body, cancellationToken).ConfigureAwait(false);
+                    if (fileEntity.Length != null)
+                        responseHeaders.ContentLength = fileEntity.Length;
+
+                    // Taken from Microsoft.AspNetCore.Internal.FileResultHelper.WriteFileAsync
+                    try
+                    {
+                        await StreamCopyOperation.CopyToAsync(fileEntity.Data, _response.Body, count: null, bufferSize: 64 * 1024, cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        _response.HttpContext.Abort();
+                    }
                     break;
                 }
 

src/Dibix.Http.Server.AspNet/HttpResponseMessageFormatter.cs

@@ -35,10 +35,16 @@ private static object CreateFileResponse(object result, HttpRequestMessageDescri
             switch (result)
             {
                 case FileEntity fileEntity:
+                {
                     mediaType = MimeTypes.IsRegistered(fileEntity.Type) ? fileEntity.Type : MimeTypes.GetMimeType(fileEntity.Type);
                     fileName = fileEntity.FileName;
-                    content = new ByteArrayContent(fileEntity.Data);
+                    content = new StreamContent(fileEntity.Data);
+
+                    if (fileEntity.Length != null)
+                        content.Headers.ContentLength = fileEntity.Length;
+
                     break;
+                }
 
 #if NETFRAMEWORK
                 case IJsonFileMetadata jsonFileMetadata:

src/Dibix.Http.Server/Runtime/HttpActionInvokerBase.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -41,9 +42,18 @@ private static async Task<object> InvokeCore<TRequest>(HttpActionDefinition acti
                 }
             }
 
-            object result = await Execute(action, request, arguments, controllerActivator, parameterDependencyResolver, cancellationToken).ConfigureAwait(false);
-            object formattedResult = await responseFormatter.Format(result, request, action, cancellationToken).ConfigureAwait(false);
-            return formattedResult;
+            object result = null;
+            try
+            {
+                result = await Execute(action, request, arguments, controllerActivator, parameterDependencyResolver, cancellationToken).ConfigureAwait(false);
+                object formattedResult = await responseFormatter.Format(result, request, action, cancellationToken).ConfigureAwait(false);
+                return formattedResult;
+            }
+            finally
+            {
+                if (result is IDisposable disposable)
+                    disposable.Dispose();
+            }
         }
 
         private static async Task<object> Execute(IHttpActionExecutionDefinition definition, IHttpRequestDescriptor request, IDictionary<string, object> arguments, IControllerActivator controllerActivator, IParameterDependencyResolver parameterDependencyResolver, CancellationToken cancellationToken)

src/Dibix.Sdk.CodeGeneration/Output/DaoExecutorWriter.cs

@@ -330,7 +330,7 @@ private static void WriteSimpleResult(StringWriter writer, SqlStatementDefinitio
 
         private static void WriteSimpleMethodCall(StringWriter writer, SqlStatementDefinition definition, SqlQueryResult singleResult, CodeGenerationContext context)
         {
-            string methodName = singleResult != null ? GetExecutorMethodName(singleResult.ResultMode) : "Execute";
+            string methodName = GetExecutorMethodName(definition, singleResult);
             WriteMethodCall(writer, definition, methodName, singleResult, context);
         }
 
@@ -482,7 +482,7 @@ private static void WriteMethodCall(StringWriter writer, SqlStatementDefinition
             if (definition.Async)
                 writer.WriteRaw("Async");
 
-            if (singleResult != null)
+            if (singleResult != null && definition.FileResult == null)
                 WriteGenericTypeArguments(writer, singleResult, context);
 
             ICollection<string> parameters = new Collection<string>();
@@ -556,6 +556,20 @@ private static void WriteOutputParameterAssignment(StringWriter writer, SqlState
             }
         }
 
+        private static string GetExecutorMethodName(SqlStatementDefinition definition, SqlQueryResult singleResult)
+        {
+            if (definition.FileResult != null && definition.IsJsonFileResult == null)
+            {
+                return "QueryFile";
+            }
+
+            if (singleResult != null)
+            {
+                return GetExecutorMethodName(singleResult.ResultMode);
+            }
+
+            return "Execute";
+        }
         private static string GetExecutorMethodName(SqlQueryResultMode mode)
         {
             switch (mode)

src/Dibix.Sdk.CodeGeneration/Parser/StatementOutputParser.cs

@@ -65,7 +65,6 @@ private static SqlQueryResult CreateBuiltInResult(SchemaDefinition schema, ISqlE
         }
         private static SqlQueryResult CreateBuiltInResult(TypeReference typeReference) => new SqlQueryResult
         {
-            ResultMode = SqlQueryResultMode.SingleOrDefault,
             Types = { typeReference },
             ReturnType = typeReference
         };
@@ -77,33 +76,52 @@ private static void ValidateFileResult(SqlStatementDefinition definition, TSqlFr
                 logger.LogError("When using the @FileResult option, no return declarations should be defined", source, node.StartLine, node.StartColumn);
             }
 
-            if (!IsValidFileResult(results))
+            if (definition.MergeGridResult)
             {
-                logger.LogError("When using the @FileResult option, the query should return only one output statement with the following schema: ([type] NVARCHAR, [data] VARBINARY, [filename] NVARCHAR NULL)", source, node.StartLine, node.StartColumn);
+                logger.LogError("When using the @FileResult option, the @MergeGridResult option is invalid", source, node.StartLine, node.StartColumn);
             }
 
-            if (definition.MergeGridResult)
+            if (results.Count != 1)
             {
-                logger.LogError("When using the @FileResult option, the @MergeGridResult option is invalid", source, node.StartLine, node.StartColumn);
+                logger.LogError("When using the @FileResult option, there should only be a single SELECT statement", source, node.StartLine, node.StartColumn);
+            }
+
+            OutputSelectResult select = results[0];
+            _ = ValidateFileResultColumn(select, "type", isRequired: true, source, logger, expectedTypes: [SqlDataType.NVarChar, SqlDataType.NChar]);
+            OutputColumnResult dataColumn = ValidateFileResultColumn(select, "data", isRequired: true, source, logger, expectedTypes: SqlDataType.VarBinary);
+            _ = ValidateFileResultColumn(select, "filename", isRequired: false, source, logger, expectedTypes: [SqlDataType.NVarChar, SqlDataType.NChar]);
+            _ = ValidateFileResultColumn(select, "length", isRequired: false, source, logger, expectedTypes: SqlDataType.BigInt);
+
+            if (dataColumn != null)
+            {
+                int index = select.Columns.IndexOf(dataColumn);
+                if (index != select.Columns.Count - 1)
+                {
+                    logger.LogError("When using the @FileResult option, the column [data] should be the last column in the SELECT", source, dataColumn.PrimarySource.StartLine, dataColumn.PrimarySource.StartColumn);
+                }
             }
         }
 
-        private static bool IsValidFileResult(IList<OutputSelectResult> results)
+        private static OutputColumnResult ValidateFileResultColumn(OutputSelectResult select, string columnName, bool isRequired, string source, ILogger logger, params SqlDataType[] expectedTypes)
         {
-            if (results.Count != 1)
-                return false;
-
-            int columnCount = results[0].Columns.Count;
-            if (columnCount is < 2 or > 3)
-                return false;
+            string expectedTypesString = String.Join("/", expectedTypes.Select(x => x.ToString().ToUpperInvariant()));
 
-            ICollection<OutputColumnResult> columns = results[0].Columns;
-            bool hasRequiredColumns = columns.Any(x => ValidateColumn(x, "data", SqlDataType.Binary, SqlDataType.VarBinary))
-                                   && columns.Any(x => ValidateColumn(x, "type", SqlDataType.Char, SqlDataType.NChar, SqlDataType.VarChar, SqlDataType.NVarChar));
+            OutputColumnResult column = select.Columns.FirstOrDefault(x => x.ColumnName.ToLowerInvariant() == columnName);
+            if (column == null)
+            {
+                if (isRequired)
+                {
+                    logger.LogError($"When using the @FileResult option, the SELECT statement should return the column [{columnName}] {expectedTypesString.ToString().ToUpperInvariant()}", source, select.Line, select.Column);
+                }
+                return null;
+            }
 
-            bool isFileNameColumnValid = columnCount < 3 || columns.Any(x => ValidateColumn(x, "filename", SqlDataType.Char, SqlDataType.NChar, SqlDataType.VarChar, SqlDataType.NVarChar));
+            if (!expectedTypes.Contains(column.DataType))
+            {
+                logger.LogError($"When using the @FileResult option, the column [{columnName}] should have the type {expectedTypesString.ToString().ToUpperInvariant()}", source, column.PrimarySource.StartLine, column.PrimarySource.StartColumn);
+            }
 
-            return hasRequiredColumns && isFileNameColumnValid;
+            return column;
         }
 
         private static void ValidateJsonFileResult(SqlStatementDefinition definition, TSqlFragment node, string source, IList<ISqlElement> returnElements, IList<OutputSelectResult> results, ILogger logger)