From daf1f3646607a7a881fffc1af8c8f338c31c922c Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 08:32:05 +0530 Subject: [PATCH 1/8] fix(csharp): fix SEA protocol parity issues for metadata, errors, and catalog handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes multiple SEA driver issues found during E2E test parity testing with Thrift protocol: - Map HTTP 401/403 to AdbcStatusCode.Unauthorized in SEA client - Wrap non-ADBC exceptions in DatabricksException during OpenAsync - Handle SPARK catalog alias (SPARK → null) at connection setup, matching Thrift's HandleSparkCatalog behavior - Fix EffectiveCatalog to pass null through (all catalogs) when SPARK is explicitly requested, instead of falling back to connection default - Fix SHOW SCHEMAS IN ALL CATALOGS column order (databaseName, catalog) not (catalog, databaseName) as previously assumed - Remove catalog filter from GetCatalogs (Thrift has no filter either) - Lowercase GetObjects patterns for case-insensitive matching - Fix Scale/Precision null-setting: null Scale for CHAR/VARCHAR types (only DECIMAL/NUMERIC should retain Scale) - Add SetOption support for TraceParent on both connection and statement - Default ExecuteUpdate AffectedRows to -1 when manifest has no count - Set _metadataCatalogName from connection default in statement constructor Co-Authored-By: Claude Opus 4.6 (1M context) --- .../StatementExecutionClient.cs | 15 +++- .../StatementExecutionConnection.cs | 77 +++++++++++++++---- .../StatementExecutionStatement.cs | 52 +++++++++++-- 3 files changed, 121 insertions(+), 23 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionClient.cs b/csharp/src/StatementExecution/StatementExecutionClient.cs index ddc52d34..d4796921 100644 --- a/csharp/src/StatementExecution/StatementExecutionClient.cs +++ b/csharp/src/StatementExecution/StatementExecutionClient.cs @@ -22,6 +22,7 @@ using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using Apache.Arrow.Adbc; namespace AdbcDrivers.Databricks.StatementExecution { @@ -403,7 +404,19 @@ private async Task EnsureSuccessStatusCodeAsync(HttpResponseMessage response) errorMessage = $"{errorMessage}. Response: {errorContent}"; } - throw new DatabricksException(errorMessage); + var statusCode = response.StatusCode switch + { + System.Net.HttpStatusCode.Unauthorized or System.Net.HttpStatusCode.Forbidden + => AdbcStatusCode.Unauthorized, + System.Net.HttpStatusCode.NotFound + => AdbcStatusCode.NotFound, + System.Net.HttpStatusCode.Conflict + => AdbcStatusCode.AlreadyExists, + System.Net.HttpStatusCode.BadRequest + => AdbcStatusCode.InvalidArgument, + _ => AdbcStatusCode.IOError, + }; + throw new DatabricksException(errorMessage, statusCode); } } } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index c49c69dd..7f74a382 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -197,6 +197,9 @@ private StatementExecutionConnection( if (_enableMultipleCatalogSupport) { properties.TryGetValue(AdbcOptions.Connection.CurrentCatalog, out _catalog); + // Match Thrift behavior: SPARK is a legacy alias — map it to null so the + // runtime falls back to the workspace default (typically hive_metastore). + _catalog = DatabricksConnection.HandleSparkCatalog(_catalog); } properties.TryGetValue(AdbcOptions.Connection.CurrentDbSchema, out _schema); @@ -363,8 +366,22 @@ public async Task OpenAsync(CancellationToken cancellationToken = default) SessionConfigs = sessionConfigs.Count > 0 ? sessionConfigs : null }; - var response = await _client.CreateSessionAsync(request, cancellationToken).ConfigureAwait(false); - _sessionId = response.SessionId; + try + { + var response = await _client.CreateSessionAsync(request, cancellationToken).ConfigureAwait(false); + _sessionId = response.SessionId; + } + catch (DatabricksException) + { + throw; + } + catch (Exception ex) + { + throw new DatabricksException( + $"Failed to connect to Databricks: {ex.GetBaseException().Message}", + AdbcStatusCode.IOError, + ex); + } // If user didn't specify a catalog, discover the server's default. // In Thrift, the server returns this in OpenSessionResp.InitialNamespace. @@ -405,6 +422,18 @@ public override AdbcStatement CreateStatement() this); // Pass connection as TracingConnection for tracing support } + public override void SetOption(string key, string? value) + { + switch (key) + { + case AdbcOptions.Telemetry.TraceParent: + SetTraceParent(string.IsNullOrWhiteSpace(value) ? null : value); + return; + } + + base.SetOption(key, value); + } + public override IArrowArrayStream GetObjects(GetObjectsDepth depth, string? catalogPattern, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes, string? columnNamePattern) { return this.TraceActivity(activity => @@ -415,6 +444,13 @@ public override IArrowArrayStream GetObjects(GetObjectsDepth depth, string? cata activity?.SetTag("table_pattern", tableNamePattern ?? "(none)"); activity?.SetTag("column_pattern", columnNamePattern ?? "(none)"); + // Databricks identifiers are case-insensitive — lowercase patterns + // to match server behavior (same as DatabricksConnection/Thrift path). + catalogPattern = catalogPattern?.ToLower(); + schemaPattern = schemaPattern?.ToLower(); + tableNamePattern = tableNamePattern?.ToLower(); + columnNamePattern = columnNamePattern?.ToLower(); + using var cts = CreateMetadataTimeoutCts(); return GetObjectsResultBuilder.BuildGetObjectsResultAsync( this, depth, catalogPattern, schemaPattern, @@ -540,7 +576,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); var batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: catalog, databaseName + // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName bool showSchemasInAllCatalogs = catalogPattern == null; @@ -552,8 +588,8 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin if (showSchemasInAllCatalogs) { - catalogArray = batch.Column(0) as StringArray; - schemaArray = batch.Column(1) as StringArray; + schemaArray = batch.Column(0) as StringArray; + catalogArray = batch.Column(1) as StringArray; } else { @@ -661,20 +697,25 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter tableInfo, colName, colType, position, nullable); // Match Thrift GetObjects behavior: SparkConnection.SetPrecisionScaleAndTypeName - // only sets Precision/Scale for DECIMAL, NUMERIC, CHAR, NCHAR, VARCHAR, - // NVARCHAR, LONGVARCHAR, LONGNVARCHAR. All other types get null. + // sets Precision for DECIMAL, NUMERIC, CHAR, NCHAR, VARCHAR, NVARCHAR, + // LONGVARCHAR, LONGNVARCHAR. Sets Scale only for DECIMAL/NUMERIC. + // All other types get null for both. int lastIdx = tableInfo.Precision.Count - 1; short typeCode = tableInfo.ColType[lastIdx]; - if (typeCode != (short)HiveServer2Connection.ColumnTypeId.DECIMAL - && typeCode != (short)HiveServer2Connection.ColumnTypeId.NUMERIC - && typeCode != (short)HiveServer2Connection.ColumnTypeId.CHAR - && typeCode != (short)HiveServer2Connection.ColumnTypeId.NCHAR - && typeCode != (short)HiveServer2Connection.ColumnTypeId.VARCHAR - && typeCode != (short)HiveServer2Connection.ColumnTypeId.NVARCHAR - && typeCode != (short)HiveServer2Connection.ColumnTypeId.LONGVARCHAR - && typeCode != (short)HiveServer2Connection.ColumnTypeId.LONGNVARCHAR) + bool isDecimalOrNumeric = typeCode == (short)HiveServer2Connection.ColumnTypeId.DECIMAL + || typeCode == (short)HiveServer2Connection.ColumnTypeId.NUMERIC; + bool isCharType = typeCode == (short)HiveServer2Connection.ColumnTypeId.CHAR + || typeCode == (short)HiveServer2Connection.ColumnTypeId.NCHAR + || typeCode == (short)HiveServer2Connection.ColumnTypeId.VARCHAR + || typeCode == (short)HiveServer2Connection.ColumnTypeId.NVARCHAR + || typeCode == (short)HiveServer2Connection.ColumnTypeId.LONGVARCHAR + || typeCode == (short)HiveServer2Connection.ColumnTypeId.LONGNVARCHAR; + if (!isDecimalOrNumeric && !isCharType) { tableInfo.Precision[lastIdx] = null; + } + if (!isDecimalOrNumeric) + { tableInfo.Scale[lastIdx] = null; } } @@ -750,6 +791,12 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc /// /// Queries the server for the current catalog via SELECT CURRENT_CATALOG(). /// + /// + /// Returns the session's default catalog. Used by statements when + /// enableMultipleCatalogSupport=false and no catalog was specified. + /// + internal string? GetSessionDefaultCatalog() => GetCurrentCatalog(); + private string? GetCurrentCatalog() { var batches = ExecuteMetadataSql("SELECT CURRENT_CATALOG()"); diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index ac6ed1a7..a82e4fe7 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -119,6 +119,11 @@ public StatementExecutionStatement( _recyclableMemoryStreamManager = recyclableMemoryStreamManager ?? throw new ArgumentNullException(nameof(recyclableMemoryStreamManager)); _lz4BufferPool = lz4BufferPool ?? throw new ArgumentNullException(nameof(lz4BufferPool)); _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + + // Match Thrift: statement starts with connection's default catalog. + // When enableMultipleCatalogSupport=true, this is the catalog from config (e.g. "main"). + // When false, _catalog is null (not set from config), matching Thrift behavior. + _metadataCatalogName = catalog; } /// @@ -184,6 +189,10 @@ public override void SetOption(string key, string value) case DatabricksParameters.MaxBytesPerFetchRequest: break; + case AdbcOptions.Telemetry.TraceParent: + SetTraceParent(string.IsNullOrEmpty(value) ? null : value); + break; + default: base.SetOption(key, value); break; @@ -603,8 +612,9 @@ public async Task ExecuteUpdateAsync(CancellationToken cancellatio throw new AdbcException("Statement was closed before results could be retrieved"); } - // For updates, we don't need to read the results - just return the row count - long rowCount = response.Manifest?.TotalRowCount ?? 0; + // For updates, we don't need to read the results - just return the row count. + // Default to -1 (unknown) when no manifest/row count, matching Thrift behavior for DDL. + long rowCount = response.Manifest?.TotalRowCount ?? -1; return new UpdateResult(rowCount); } @@ -779,7 +789,33 @@ private static async Task FetchAllChunksAsync( // Metadata command routing - private string? EffectiveCatalog => _connection.ResolveEffectiveCatalog(_metadataCatalogName); + /// + /// Resolves the catalog for metadata SQL commands. + /// Matches Thrift behavior: + /// - SPARK → null (all catalogs) + /// - Other values pass through as-is + /// - null stays null + /// When enableMultipleCatalogSupport=false and result is null, + /// resolves to the session default catalog (SEA SQL requires an explicit + /// catalog for SHOW commands when not querying all catalogs). + /// + private string? EffectiveCatalog + { + get + { + // Normalize SPARK → null, same as Thrift's HandleSparkCatalog + string? catalog = DatabricksConnection.HandleSparkCatalog(_metadataCatalogName); + + if (_connection.EnableMultipleCatalogSupport) + { + // null means "all catalogs" (e.g. SHOW SCHEMAS IN ALL CATALOGS) + return catalog; + } + + // flag=false: null means use session default (SEA SQL needs explicit catalog) + return catalog ?? _connection.GetSessionDefaultCatalog(); + } + } /// /// Escapes wildcard characters (_ and %) in metadata name parameters when @@ -824,7 +860,9 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT return new QueryResult(1, new HiveInfoArrowStream(catalogSchema, new IArrowArray[] { sparkBuilder.Build() })); } - string sql = new ShowCatalogsCommand(EscapePatternWildcardsInName(_metadataCatalogName)).Build(); + // GetCatalogs returns all catalogs — no filtering by pattern, + // matching Thrift behavior (Thrift RPC has no catalog filter for GetCatalogs). + string sql = new ShowCatalogsCommand(null).Build(); activity?.SetTag("sql_query", sql); var batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); @@ -870,7 +908,7 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo activity?.SetTag("sql_query", sql); var batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: catalog_name, databaseName + // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName bool showAllCatalogs = catalog == null; @@ -884,8 +922,8 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo if (showAllCatalogs) { - catalogArray = batch.Column(0) as StringArray; - schemaArray = batch.Column(1) as StringArray; + schemaArray = batch.Column(0) as StringArray; + catalogArray = batch.Column(1) as StringArray; } else { From e81dc755deee01ec57dce551adba1d0903a80e73 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 09:31:03 +0530 Subject: [PATCH 2/8] fix: separate XML doc comments for GetSessionDefaultCatalog and GetCurrentCatalog Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/StatementExecution/StatementExecutionConnection.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 90f2cc7b..2fc5e7f7 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -805,15 +805,15 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc return normalized ?? GetCurrentCatalog(); } - /// - /// Queries the server for the current catalog via SELECT CURRENT_CATALOG(). - /// /// /// Returns the session's default catalog. Used by statements when /// enableMultipleCatalogSupport=false and no catalog was specified. /// internal string? GetSessionDefaultCatalog() => GetCurrentCatalog(); + /// + /// Queries the server for the current catalog via SELECT CURRENT_CATALOG(). + /// private string? GetCurrentCatalog() { var batches = ExecuteMetadataSql("SELECT CURRENT_CATALOG()"); From da40538ed15447b6d327512a50218cb96125260e Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 11:02:08 +0530 Subject: [PATCH 3/8] fix: return null Scale for non-DECIMAL types in GetDecimalDigitsDefault GetDecimalDigitsDefault now only returns a value for DECIMAL/NUMERIC. All other types return null, so Scale is correctly null from the start in PopulateTableInfoFromTypeName. This removes the need for post-processing Scale null-setting in PopulateColumnInfoAsync. Simplified the post-processing to only null out Precision for types that don't need it (non-DECIMAL/CHAR). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../StatementExecutionConnection.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 2fc5e7f7..62c7c987 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -713,10 +713,13 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter ColumnMetadataHelper.PopulateTableInfoFromTypeName( tableInfo, colName, colType, position, nullable); - // Match Thrift GetObjects behavior: SparkConnection.SetPrecisionScaleAndTypeName - // sets Precision for DECIMAL, NUMERIC, CHAR, NCHAR, VARCHAR, NVARCHAR, - // LONGVARCHAR, LONGNVARCHAR. Sets Scale only for DECIMAL/NUMERIC. - // All other types get null for both. + // For GetObjects, null out Precision and Scale for types where Thrift + // returns null (see SparkConnection.SetPrecisionScaleAndTypeName): + // - Precision: only DECIMAL/NUMERIC and CHAR/VARCHAR retain it + // - Scale: only DECIMAL/NUMERIC retains it + // Note: GetColumnSizeDefault/GetDecimalDigitsDefault return non-null for + // all types (matching GetColumns server values), so we null them here + // specifically for the GetObjects path. int lastIdx = tableInfo.Precision.Count - 1; short typeCode = tableInfo.ColType[lastIdx]; bool isDecimalOrNumeric = typeCode == (short)HiveServer2Connection.ColumnTypeId.DECIMAL From 307a0a51f97fddafd1dd2190cdf631bfac94b9ac Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 13:30:05 +0530 Subject: [PATCH 4/8] fix: fallback for SHOW COLUMNS when catalog is null (all catalogs) SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend. When catalog is null, iterate over all catalogs via SHOW CATALOGS and call SHOW COLUMNS IN `catalog` for each, merging results. Catalogs that return permission errors are silently skipped. Both GetColumnsAsync (statement) and PopulateColumnInfoAsync (connection/GetObjects) use the shared ExecuteShowColumnsAsync helper. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../StatementExecutionConnection.cs | 46 ++++++++++++++++++- .../StatementExecutionStatement.cs | 7 ++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 62c7c987..78f2c836 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -668,8 +668,7 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - string sql = new ShowColumnsCommand(catalogPattern, schemaPattern, tablePattern, columnPattern).Build(); - var batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var batches = await ExecuteShowColumnsAsync(catalogPattern, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false); var tablePositions = new Dictionary(); @@ -778,6 +777,49 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc return ExecuteMetadataSqlAsync(sql, cancellationToken).GetAwaiter().GetResult(); } + /// + /// Executes a SHOW COLUMNS command. When catalog is null, iterates over all catalogs + /// since SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend. + /// + internal async Task> ExecuteShowColumnsAsync( + string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, + CancellationToken cancellationToken) + { + if (catalog != null) + { + string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); + return await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + } + + // SHOW COLUMNS IN ALL CATALOGS is not supported — iterate over each catalog. + // TODO: Remove this fallback when the backend supports SHOW COLUMNS IN ALL CATALOGS. + var allBatches = new List(); + string catalogsSql = new ShowCatalogsCommand(null).Build(); + var catalogBatches = await ExecuteMetadataSqlAsync(catalogsSql, cancellationToken).ConfigureAwait(false); + + foreach (var batch in catalogBatches) + { + var catalogArray = batch.Column(0) as StringArray; + if (catalogArray == null) continue; + for (int i = 0; i < catalogArray.Length; i++) + { + if (catalogArray.IsNull(i)) continue; + string cat = catalogArray.GetString(i); + string sql = new ShowColumnsCommand(cat, schemaPattern, tablePattern, columnPattern).Build(); + try + { + var batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + allBatches.AddRange(batches); + } + catch + { + // Skip catalogs we can't access (permission errors) + } + } + } + return allBatches; + } + internal bool EnablePKFK => _enablePKFK; internal bool EnableMultipleCatalogSupport => _enableMultipleCatalogSupport; diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 7dfe94e5..11c979d7 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1061,13 +1061,12 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - string sql = new ShowColumnsCommand( + var batches = await _connection.ExecuteShowColumnsAsync( catalog, EscapePatternWildcardsInName(_metadataSchemaName), EscapePatternWildcardsInName(_metadataTableName), - EscapePatternWildcardsInName(_metadataColumnName)).Build(); - activity?.SetTag("sql_query", sql); - var batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + EscapePatternWildcardsInName(_metadataColumnName), + cancellationToken).ConfigureAwait(false); var tableInfos = new Dictionary(); From c9847c0e0f2aead5c467b0b46966d530a3cc833c Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 13:44:27 +0530 Subject: [PATCH 5/8] fix: remove Precision/Scale post-processing in GetObjects PopulateColumnInfoAsync Keep GetColumnSizeDefault and GetDecimalDigitsDefault values for all types in GetObjects, matching GetColumns/GetColumnsExtended behavior. This diverges from Thrift's basic GetObjects (which nulls Precision/Scale for non-DECIMAL/CHAR types) but provides more useful metadata and consistent values across all metadata APIs. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../StatementExecutionConnection.cs | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 7a1faf69..670134d6 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -713,32 +713,6 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter { ColumnMetadataHelper.PopulateTableInfoFromTypeName( tableInfo, colName, colType, position, nullable); - - // For GetObjects, null out Precision and Scale for types where Thrift - // returns null (see SparkConnection.SetPrecisionScaleAndTypeName): - // - Precision: only DECIMAL/NUMERIC and CHAR/VARCHAR retain it - // - Scale: only DECIMAL/NUMERIC retains it - // Note: GetColumnSizeDefault/GetDecimalDigitsDefault return non-null for - // all types (matching GetColumns server values), so we null them here - // specifically for the GetObjects path. - int lastIdx = tableInfo.Precision.Count - 1; - short typeCode = tableInfo.ColType[lastIdx]; - bool isDecimalOrNumeric = typeCode == (short)HiveServer2Connection.ColumnTypeId.DECIMAL - || typeCode == (short)HiveServer2Connection.ColumnTypeId.NUMERIC; - bool isCharType = typeCode == (short)HiveServer2Connection.ColumnTypeId.CHAR - || typeCode == (short)HiveServer2Connection.ColumnTypeId.NCHAR - || typeCode == (short)HiveServer2Connection.ColumnTypeId.VARCHAR - || typeCode == (short)HiveServer2Connection.ColumnTypeId.NVARCHAR - || typeCode == (short)HiveServer2Connection.ColumnTypeId.LONGVARCHAR - || typeCode == (short)HiveServer2Connection.ColumnTypeId.LONGNVARCHAR; - if (!isDecimalOrNumeric && !isCharType) - { - tableInfo.Precision[lastIdx] = null; - } - if (!isDecimalOrNumeric) - { - tableInfo.Scale[lastIdx] = null; - } } } } From f7694ed748ce6713f797c609abf570551b751a64 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 13:51:02 +0530 Subject: [PATCH 6/8] chore: add TODO to consolidate EffectiveCatalog with ResolveEffectiveCatalog Co-Authored-By: Claude Opus 4.6 (1M context) --- csharp/src/StatementExecution/StatementExecutionStatement.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 01676dbc..1af940c3 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -812,6 +812,9 @@ private static async Task FetchAllChunksAsync( /// resolves to the session default catalog (SEA SQL requires an explicit /// catalog for SHOW commands when not querying all catalogs). /// + // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, consider + // consolidating with StatementExecutionConnection.ResolveEffectiveCatalog + // so both statement and connection use the same catalog resolution logic. private string? EffectiveCatalog { get From 81a779a3194494394aa13ad6dad0e4c3fc2dffb2 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 14:30:17 +0530 Subject: [PATCH 7/8] refactor: remove ResolveEffectiveCatalog, use ExecuteShowColumnsAsync in GetTableSchema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetTableSchema now uses ExecuteShowColumnsAsync with HandleSparkCatalog, matching Thrift behavior which passes catalog as-is to the server (null = all catalogs, SPARK = null). Removed ResolveEffectiveCatalog since it's no longer needed — all catalog resolution now uses either: - Statement's EffectiveCatalog (for metadata commands) - HandleSparkCatalog + ExecuteShowColumnsAsync (for GetTableSchema) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../StatementExecutionConnection.cs | 36 ++++--------------- .../StatementExecutionStatement.cs | 5 ++- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 670134d6..7c865b79 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -532,10 +532,12 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string activity?.SetTag("table_name", tableName); using var cts = CreateMetadataTimeoutCts(); - string sql = new ShowColumnsCommand( - ResolveEffectiveCatalog(catalog), dbSchema, tableName).Build(); - activity?.SetTag("sql_query", sql); - var batches = ExecuteMetadataSql(sql, cts.Token); + // Pass catalog through with SPARK→null normalization, matching Thrift + // which sends catalog as-is to the server. ExecuteShowColumnsAsync + // handles null by iterating all catalogs. + string? resolvedCatalog = DatabricksConnection.HandleSparkCatalog(catalog); + var batches = ExecuteShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) + .GetAwaiter().GetResult(); var fields = new List(); foreach (var batch in batches) @@ -807,32 +809,6 @@ internal async Task> ExecuteShowColumnsAsync( /// internal bool UseDescTableExtended => _useDescTableExtended; - /// - /// Resolves the effective catalog for metadata queries. - /// SEA SHOW commands require an explicit catalog name in the SQL string - /// (e.g., SHOW SCHEMAS IN `catalog`), unlike Thrift which treats null - /// as "use session default." So we must always resolve to a concrete value. - /// When EnableMultipleCatalogSupport is true: uses the provided catalog, - /// falling back to the connection default catalog. - /// When EnableMultipleCatalogSupport is false: resolves via the session's - /// current catalog (SELECT CURRENT_CATALOG()) since _catalog is null - /// when the flag is false (matching Thrift behavior). - /// - internal string? ResolveEffectiveCatalog(string? requestedCatalog) - { - string? normalized = MetadataUtilities.NormalizeSparkCatalog(requestedCatalog); - - if (_enableMultipleCatalogSupport) - { - return normalized ?? _catalog; - } - - // flag=false: if user specified an explicit non-null catalog, it won't - // match the default — the statement layer should return empty. - // If null/SPARK, resolve via server query. - return normalized ?? GetCurrentCatalog(); - } - /// /// Returns the session's default catalog. Used by statements when /// enableMultipleCatalogSupport=false and no catalog was specified. diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 1af940c3..97a23424 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -812,9 +812,8 @@ private static async Task FetchAllChunksAsync( /// resolves to the session default catalog (SEA SQL requires an explicit /// catalog for SHOW commands when not querying all catalogs). /// - // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, consider - // consolidating with StatementExecutionConnection.ResolveEffectiveCatalog - // so both statement and connection use the same catalog resolution logic. + // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, the + // ExecuteShowColumnsAsync iterate-all-catalogs fallback can be removed. private string? EffectiveCatalog { get From edcd170871adc66b16bc54b109e4d7ec0a61b479 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Thu, 12 Mar 2026 14:52:01 +0530 Subject: [PATCH 8/8] fix: revert AffectedRows/RowCount default back to 0 Reverts the ?? -1 change back to ?? 0 for both ExecuteQuery and ExecuteUpdate. SEA API returns 0 for DDL, not -1. Keeping original SEA behavior. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/StatementExecution/StatementExecutionStatement.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 97a23424..2c19d2f8 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -304,8 +304,8 @@ public async Task ExecuteQueryAsync( // Get schema from reader var schema = reader.Schema; - // Return query result - use -1 if row count is not available - long rowCount = response.Manifest?.TotalRowCount ?? -1; + // Return query result - use 0 if row count is not available + long rowCount = response.Manifest?.TotalRowCount ?? 0; return new QueryResult(rowCount, reader); } @@ -626,8 +626,7 @@ public async Task ExecuteUpdateAsync(CancellationToken cancellatio } // For updates, we don't need to read the results - just return the row count. - // Default to -1 (unknown) when no manifest/row count, matching Thrift behavior for DDL. - long rowCount = response.Manifest?.TotalRowCount ?? -1; + long rowCount = response.Manifest?.TotalRowCount ?? 0; return new UpdateResult(rowCount); }