feat(csharp): improve handling of StructArrays

csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs

@@ -19,12 +19,19 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Data.SqlTypes;
+using System.Dynamic;
 using System.IO;
 using System.Text.Json;
 using Apache.Arrow.Types;
 
 namespace Apache.Arrow.Adbc.Extensions
 {
+    public enum StructResultType
+    {
+        JsonString,
+        Object
+    }
+
     public static class IArrowArrayExtensions
     {
         /// <summary>
@@ -36,7 +43,7 @@ public static class IArrowArrayExtensions
         /// <param name="index">
         /// The index in the array to get the value from.
         /// </param>
-        public static object? ValueAt(this IArrowArray arrowArray, int index)
+        public static object? ValueAt(this IArrowArray arrowArray, int index, StructResultType resultType = StructResultType.JsonString)
         {
             if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray));
             if (index < 0) throw new ArgumentOutOfRangeException(nameof(index));
@@ -76,7 +83,9 @@ public static class IArrowArrayExtensions
                 case ArrowTypeId.Int64:
                     return ((Int64Array)arrowArray).GetValue(index);
                 case ArrowTypeId.String:
-                    return ((StringArray)arrowArray).GetString(index);
+                    StringArray sArray = (StringArray)arrowArray;
+                    if (sArray.Length == 0) { return null; }
+                    return sArray.GetString(index);
 #if NET6_0_OR_GREATER
                 case ArrowTypeId.Time32:
                     return ((Time32Array)arrowArray).GetTime(index);
@@ -138,13 +147,14 @@ public static class IArrowArrayExtensions
                 case ArrowTypeId.List:
                     return ((ListArray)arrowArray).GetSlicedValues(index);
                 case ArrowTypeId.Struct:
-                    return SerializeToJson(((StructArray)arrowArray), index);
+                    StructArray structArray = (StructArray)arrowArray;
+                    return resultType == StructResultType.JsonString ? SerializeToJson(structArray, index) : ParseStructArray(structArray, index);
 
-                // not covered:
-                // -- map array
-                // -- dictionary array
-                // -- fixed size binary
-                // -- union array
+                    // not covered:
+                    // -- map array
+                    // -- dictionary array
+                    // -- fixed size binary
+                    // -- union array
             }
 
             return null;
@@ -159,7 +169,7 @@ public static class IArrowArrayExtensions
         /// <param name="index">
         /// The index in the array to get the value from.
         /// </param>
-        public static Func<IArrowArray, int, object?> GetValueConverter(this IArrowType arrayType)
+        public static Func<IArrowArray, int, object?> GetValueConverter(this IArrowType arrayType, StructResultType resultType = StructResultType.JsonString)
         {
             if (arrayType == null) throw new ArgumentNullException(nameof(arrayType));
 
@@ -198,7 +208,23 @@ public static class IArrowArrayExtensions
                 case ArrowTypeId.Int64:
                     return (array, index) => ((Int64Array)array).GetValue(index);
                 case ArrowTypeId.String:
-                    return (array, index) => ((StringArray)array).GetString(index);
+                    return (array, index) =>
+                    {
+                        StringArray? sArray = array as StringArray;
+
+                        if (sArray != null)
+                        {
+                            return sArray.GetString(index);
+                        }
+                        else
+                        {
+                            // some callers treat the Decimal256Array values as a string
+                            Decimal256Array? array256 = array as Decimal256Array;
+                            return array256?.GetString(index);
+                        }
+
+                        throw new AdbcException($"Cannot get the value at {index}. A String type was requested but neither a StringArray or Decimal256Array was found.", AdbcStatusCode.InvalidData);
+                    };
 #if NET6_0_OR_GREATER
                 case ArrowTypeId.Time32:
                     return (array, index) => ((Time32Array)array).GetTime(index);
@@ -256,7 +282,7 @@ public static class IArrowArrayExtensions
                 case ArrowTypeId.List:
                     return (array, index) => ((ListArray)array).GetSlicedValues(index);
                 case ArrowTypeId.Struct:
-                    return (array, index) => SerializeToJson((StructArray)array, index);
+                    return (array, index) => resultType == StructResultType.JsonString ? SerializeToJson((StructArray)array, index) : ParseStructArray((StructArray)array, index) ;
 
                     // not covered:
                     // -- map array
@@ -273,20 +299,22 @@ public static class IArrowArrayExtensions
         /// </summary>
         private static string SerializeToJson(StructArray structArray, int index)
         {
-            Dictionary<String, object?>? jsonDictionary = ParseStructArray(structArray, index);
+            ExpandoObject? obj = ParseStructArray(structArray, index);
 
-            return JsonSerializer.Serialize(jsonDictionary);
+            return JsonSerializer.Serialize(obj);
         }
 
         /// <summary>
-        /// Converts a StructArray to a Dictionary<String, object?>.
+        /// Converts an item in the StructArray at the index position to an ExpandoObject.
         /// </summary>
-        private static Dictionary<String, object?>? ParseStructArray(StructArray structArray, int index)
+        private static ExpandoObject? ParseStructArray(StructArray structArray, int index)
         {
             if (structArray.IsNull(index))
                 return null;
 
-            Dictionary<String, object?> jsonDictionary = new Dictionary<String, object?>();
+            var expando = new ExpandoObject();
+            var jsonDictionary = (IDictionary<string, object?>)expando;
+
             StructType structType = (StructType)structArray.Data.DataType;
             for (int i = 0; i < structArray.Data.Children.Length; i++)
             {
@@ -295,7 +323,7 @@ private static string SerializeToJson(StructArray structArray, int index)
 
                 if (value is StructArray structArray1)
                 {
-                    List<Dictionary<string, object?>?> children = new List<Dictionary<string, object?>?>();
+                    List<ExpandoObject?> children = new List<ExpandoObject?>();
 
                     for (int j = 0; j < structArray1.Length; j++)
                     {
@@ -335,7 +363,7 @@ private static string SerializeToJson(StructArray structArray, int index)
                 }
             }
 
-            return jsonDictionary;
+            return expando;
         }
 
         /// <summary>

csharp/src/Client/AdbcDataReader.cs

@@ -86,10 +86,12 @@ internal AdbcDataReader(AdbcCommand adbcCommand, QueryResult adbcQueryResult, De
             this.DecimalBehavior = decimalBehavior;
             this.StructBehavior = structBehavior;
 
+            StructResultType structResultType = this.StructBehavior == StructBehavior.JsonString ? StructResultType.JsonString : StructResultType.Object;
+
             this.converters = new Func<IArrowArray, int, object?>[this.schema.FieldsList.Count];
             for (int i = 0; i < this.converters.Length; i++)
             {
-                this.converters[i] = this.schema.FieldsList[i].DataType.GetValueConverter();
+                this.converters[i] = this.schema.FieldsList[i].DataType.GetValueConverter(structResultType);
             }
         }
 
@@ -372,7 +374,9 @@ public ReadOnlyCollection<AdbcColumn> GetAdbcColumnSchema()
                 }
                 else
                 {
-                    dbColumns.Add(new AdbcColumn(f.Name, t, f.DataType, f.IsNullable));
+                    IArrowType arrowType = SchemaConverter.GetArrowTypeBasedOnRequestedBehavior(f.DataType, this.StructBehavior);
+
+                    dbColumns.Add(new AdbcColumn(f.Name, t, arrowType, f.IsNullable));
                 }
             }
 

csharp/src/Client/SchemaConverter.cs

@@ -19,6 +19,7 @@
 using System.Data;
 using System.Data.Common;
 using System.Data.SqlTypes;
+using System.Dynamic;
 using Apache.Arrow.Scalars;
 using Apache.Arrow.Types;
 
@@ -60,7 +61,7 @@ public static DataTable ConvertArrowSchema(Schema schema, AdbcStatement adbcStat
                 row[SchemaTableColumn.ColumnName] = f.Name;
                 row[SchemaTableColumn.ColumnOrdinal] = columnOrdinal;
                 row[SchemaTableColumn.AllowDBNull] = f.IsNullable;
-                row[SchemaTableColumn.ProviderType] = f.DataType;
+                row[SchemaTableColumn.ProviderType] = SchemaConverter.GetArrowTypeBasedOnRequestedBehavior(f.DataType, structBehavior);
                 Type t = ConvertArrowType(f, decimalBehavior, structBehavior);
 
                 row[SchemaTableColumn.DataType] = t;
@@ -193,10 +194,7 @@ public static Type GetArrowType(Field f, DecimalBehavior decimalBehavior, Struct
                     return typeof(string);
 
                 case ArrowTypeId.Struct:
-                    if (structBehavior == StructBehavior.JsonString)
-                        return typeof(string);
-                    else
-                        goto default;
+                    return  structBehavior == StructBehavior.JsonString ? typeof(string) : typeof(ExpandoObject);
 
                 case ArrowTypeId.Timestamp:
                     return typeof(DateTimeOffset);
@@ -271,5 +269,18 @@ public static Type GetArrowArrayType(IArrowType dataType)
 
             throw new InvalidCastException($"Cannot determine the array type for {dataType.Name}");
         }
+
+        /// <summary>
+        /// Get the IArrowType based on the input IArrowType and the desired <see cref="StructBehavior"/>.
+        /// If it's a StructType and the desired behavior is a JsonString then this returns StringType.
+        /// Otherwise, it returns the input IArrowType.
+        /// </summary>
+        /// <param name="defaultType">The default IArrowType to return.</param>
+        /// <param name="structBehavior">Desired behavior if the IArrowType is a StructType.</param>
+        /// <returns></returns>
+        public static IArrowType GetArrowTypeBasedOnRequestedBehavior(IArrowType defaultType, StructBehavior structBehavior)
+        {
+            return defaultType.TypeId == ArrowTypeId.Struct && structBehavior == StructBehavior.JsonString ? StringType.Default : defaultType;
+        }
     }
 }

csharp/src/Client/StructBehavior.cs

@@ -17,6 +17,9 @@
 
 namespace Apache.Arrow.Adbc.Client
 {
+    /// <summary>
+    /// Controls the behavior of how StructArrays should be handled in the results.
+    /// </summary>
     public enum StructBehavior
     {
         /// <summary>

csharp/src/Client/readme.md

@@ -80,5 +80,5 @@ These properties are:
 
 - __AdbcConnectionTimeout__ - This specifies the connection timeout value. The value needs to be in the form (driver.property.name, integer, unit) where the unit is one of `s` or `ms`, For example, `AdbcConnectionTimeout=(adbc.snowflake.sql.client_option.client_timeout,30,s)` would set the connection timeout to 30 seconds.
 - __AdbcCommandTimeout__ - This specifies the command timeout value. This follows the same pattern as `AdbcConnectionTimeout` and sets the `AdbcCommandTimeoutProperty` and `CommandTimeout` values on the `AdbcCommand` object.
-- __StructBehavior__ - This specifies the StructBehavior when working with Arrow Struct arrays. The valid values are `JsonString` (the default) or `Strict` (treat the struct as a native type).
+- __StructBehavior__ - This specifies the StructBehavior when working with Arrow Struct arrays. The valid values are `JsonString` (the default) or `Strict` (treat the struct as a native type). If using JsonString, the return ArrowType will be StringType and a string value. If using Strict, the return ArrowType will be StructType and an ExpandoObject.
 - __DecimalBehavior__ - This specifies the DecimalBehavior when parsing decimal values from Arrow libraries. The valid values are `UseSqlDecimal` or `OverflowDecimalAsString` where values like Decimal256 are treated as strings.

csharp/src/Drivers/BigQuery/BigQueryStatement.cs

@@ -112,7 +112,13 @@ public override QueryResult ExecuteQuery()
             ReadSession rrs = readClient.CreateReadSession("projects/" + results.TableReference.ProjectId, rs, maxStreamCount);
 
             long totalRows = results.TotalRows == null ? -1L : (long)results.TotalRows.Value;
-            IArrowArrayStream stream = new MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s => ReadChunk(readClient, s.Name)));
+
+            var readers = rrs.Streams
+                             .Select(s => ReadChunk(readClient, s.Name))
+                             .Where(chunk => chunk != null)
+                             .Cast<IArrowReader>();
+
+            IArrowArrayStream stream = new MultiArrowReader(TranslateSchema(results.Schema), readers);
 
             return new QueryResult(totalRows, stream);
         }
@@ -175,8 +181,7 @@ private IArrowType TranslateType(TableFieldSchema field)
                 case "DATE":
                     return GetType(field, Date32Type.Default);
                 case "RECORD" or "STRUCT":
-                    // its a json string
-                    return GetType(field, StringType.Default);
+                    return GetType(field, BuildStructType(field));
 
                 // treat these values as strings
                 case "GEOGRAPHY" or "JSON":
@@ -200,6 +205,19 @@ private IArrowType TranslateType(TableFieldSchema field)
             }
         }
 
+        private StructType BuildStructType(TableFieldSchema field)
+        {
+            List<Field> arrowFields = new List<Field>();
+
+            foreach (TableFieldSchema subfield in field.Fields)
+            {
+                Field arrowField = TranslateField(subfield);
+                arrowFields.Add(arrowField);
+            }
+
+            return new StructType(arrowFields.AsReadOnly());
+        }
+
         private IArrowType GetType(TableFieldSchema field, IArrowType type)
         {
             if (field.Mode == "REPEATED")
@@ -208,7 +226,7 @@ private IArrowType GetType(TableFieldSchema field, IArrowType type)
             return type;
         }
 
-        static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName)
+        static IArrowReader? ReadChunk(BigQueryReadClient readClient, string streamName)
         {
             // Ideally we wouldn't need to indirect through a stream, but the necessary APIs in Arrow
             // are internal. (TODO: consider changing Arrow).
@@ -217,7 +235,14 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName)
 
             ReadRowsStream stream = new ReadRowsStream(enumerator);
 
-            return new ArrowStreamReader(stream);
+            if (stream.HasRows)
+            {
+                return new ArrowStreamReader(stream);
+            }
+            else
+            {
+                return null;
+            }
         }
 
         private QueryOptions ValidateOptions()
@@ -349,15 +374,27 @@ sealed class ReadRowsStream : Stream
             ReadOnlyMemory<byte> currentBuffer;
             bool first;
             int position;
+            bool hasRows = true;
 
             public ReadRowsStream(IAsyncEnumerator<ReadRowsResponse> response)
             {
                 if (!response.MoveNextAsync().Result) { }
-                this.currentBuffer = response.Current.ArrowSchema.SerializedSchema.Memory;
+
+                if (response.Current != null)
+                {
+                    this.currentBuffer = response.Current.ArrowSchema.SerializedSchema.Memory;
+                }
+                else
+                {
+                    this.hasRows = false;
+                }
+
                 this.response = response;
                 this.first = true;
             }
 
+            public bool HasRows { get => this.hasRows; }
+
             public override bool CanRead => true;
 
             public override bool CanSeek => false;