Support fetching TIMESTAMPTZ as SQL_TYPE_TIMESTAMP

DuckDB supports `TIMESTAMP WITH TIME ZONE` data type that stores UTC
dates (with specified time zone already applied on DB insert). ODBC
does not have a notion of time zones. When client app reads such
`TIMESTAMP WITH TIME ZONE` it needs to be converted into
`TIMESTAMP WITHOUT TIME ZONE` with client-local time zone applied. And
such converted timestamp must represent the same moment of time as
stored UTC timestamp.

Applying client-local time zone to timestamp is a non-trivial operation
 - the UTC offset depends on both the time zone (for fixed shift) and on
the timestamp value (for DST shift).

There are two options to get the effective offset for the specified UTC
timestamp in the specified time zone:

 - DuckDB ICU extension, that will use DB-configured time zone
 - OS API, will use OS-configured time zone

Variant with ICU extension appeared to be problematic, as we would like
to not link neither to ICU extension (C++ API, not in DuckDB C API) nor
to ICU itself (C API exists, but symbols include ICU major version that
can change). We can do ICU call from ODBC by creating SQL string and
executing it. But this conversion must be performed for every value in
every row. And there no reasonbly-easy ways to cache the results. Thus
the idea of SQL calls was rejected.

Variant with OS API matches the approach used in JDBC (see
duckdb/duckdb-java#166) where JVM-configured time zone is used
(invariant 1 there).

The problem with accessing OS API is that `<ctime>` utilities are not
thread-safe until `C++20`. Thus Windows `<timezoneapi.h>` and POSIX
`<time.h>` API are used directly. And `<ctime>` approach is used in
tests for cross-checking.

Testing: new test added that fetches `TIMESTAMP WITH TIME ZONE` value
as `SQL_C_TYPE_TIMESTAMP` and checks that it is shifted correctly.

Author: staticlibs

include/odbc_utils.hpp

@@ -4,6 +4,7 @@
 // needs to be first because BOOL
 #include "duckdb.hpp"
 
+#include <cstdint>
 #include <cstring>
 
 #ifdef _WIN32
@@ -93,6 +94,8 @@ struct OdbcUtils {
 	static std::string ConvertSQLCHARToString(SQLCHAR *str);
 	static LPCSTR ConvertStringToLPCSTR(const std::string &str);
 	static SQLCHAR *ConvertStringToSQLCHAR(const std::string &str);
+
+	static int64_t GetUTCOffsetMicrosFromOS(HSTMT hstmt, int64_t utc_micros);
 };
 } // namespace duckdb
 #endif

src/api_info.cpp

@@ -267,6 +267,8 @@ SQLSMALLINT ApiInfo::FindRelatedSQLType(duckdb::LogicalTypeId type_id) {
 		return SQL_TYPE_DATE;
 	case LogicalTypeId::TIMESTAMP:
 		return SQL_TYPE_TIMESTAMP;
+	case LogicalTypeId::TIMESTAMP_TZ:
+		return SQL_TYPE_TIMESTAMP;
 	case LogicalTypeId::TIME:
 		return SQL_TYPE_TIME;
 	case LogicalTypeId::VARCHAR:

src/common/odbc_utils.cpp

@@ -3,9 +3,17 @@
 #include "sqlext.h"
 
 #include <sql.h>
+
+#include <mutex>
 #include <regex>
+
+#ifndef _WIN32
+#include <time.h>
+#endif //!_WIN32
+
 #include "duckdb/common/vector.hpp"
 
+#include "handle_functions.hpp"
 #include "widechar.hpp"
 
 using duckdb::OdbcUtils;
@@ -106,7 +114,6 @@ string OdbcUtils::GetQueryDuckdbColumns(const string &catalog_filter, const stri
                 'FLOAT': 6, -- SQL_FLOAT
                 'DOUBLE': 8, -- SQL_DOUBLE
                 'DATE': 91, -- SQL_TYPE_DATE
-                'TIMESTAMP': 93, -- SQL_TYPE_TIMESTAMP
                 'TIME': 92, -- SQL_TYPE_TIME
                 'VARCHAR': 12, -- SQL_VARCHAR
                 'BLOB': -3, -- SQL_VARBINARY
@@ -117,11 +124,15 @@ string OdbcUtils::GetQueryDuckdbColumns(const string &catalog_filter, const stri
             } AS mapping,
             STRING_SPLIT(data_type, '(')[1] AS data_type_no_typmod,
             CASE
+                -- value 93 is SQL_TIMESTAMP
+                -- TODO: investigate why map key with spaces and regexes don't work
+                WHEN data_type LIKE 'TIMESTAMP%' THEN 93::SMALLINT
                 WHEN mapping[data_type_no_typmod] IS NOT NULL THEN mapping[data_type_no_typmod]::SMALLINT
                 ELSE data_type_id::SMALLINT
             END AS "DATA_TYPE",
             CASE
                 WHEN data_type_no_typmod = 'DECIMAL' THEN 'NUMERIC'
+                WHEN data_type LIKE 'TIMESTAMP%' THEN 'TIMESTAMP'
                 ELSE data_type_no_typmod
             END AS "TYPE_NAME",
             CASE
@@ -140,8 +151,8 @@ string OdbcUtils::GetQueryDuckdbColumns(const string &catalog_filter, const stri
             END AS "COLUMN_SIZE",
             CASE
                 WHEN data_type='DATE' THEN 4
-                WHEN data_type='TIME' THEN 8
                 WHEN data_type LIKE 'TIMESTAMP%' THEN 8
+                WHEN data_type LIKE 'TIME%' THEN 8 
                 WHEN data_type='CHAR'
                     OR data_type='BOOLEAN' THEN 1
                 WHEN data_type='VARCHAR'
@@ -166,13 +177,14 @@ string OdbcUtils::GetQueryDuckdbColumns(const string &catalog_filter, const stri
             '' AS "REMARKS",
             column_default AS "COLUMN_DEF",
             CASE
+                WHEN data_type LIKE 'TIMESTAMP%' THEN 93::SMALLINT
                 WHEN mapping[data_type_no_typmod] IS NOT NULL THEN mapping[data_type_no_typmod]::SMALLINT
                 ELSE data_type_id::SMALLINT
             END AS "SQL_DATA_TYPE",
             CASE
-                WHEN data_type='DATE'
-                     OR data_type='TIME'
-                     OR data_type LIKE 'TIMESTAMP%' THEN data_type_id::SMALLINT
+                WHEN data_type='DATE' then 1::SMALLINT
+                WHEN data_type LIKE 'TIMESTAMP%' THEN 3::SMALLINT
+                WHEN data_type LIKE 'TIME%' THEN 2::SMALLINT
                 ELSE NULL::SMALLINT
             END AS "SQL_DATETIME_SUB",
             CASE
@@ -309,3 +321,131 @@ void duckdb::OdbcUtils::WriteString(const std::vector<SQLCHAR> &utf8_vec, std::s
                                     SQLLEN buf_len_bytes, SQLINTEGER *out_len_bytes) {
 	return WriteStringInternal(utf8_vec.data(), utf8_vec_len, out_buf, buf_len_bytes, out_len_bytes);
 }
+
+int64_t duckdb::OdbcUtils::GetUTCOffsetMicrosFromOS(HSTMT hstmt_ptr, int64_t utc_micros) {
+
+	// Casting here to not complicate header dependencies
+	auto hstmt = reinterpret_cast<duckdb::OdbcHandleStmt *>(hstmt_ptr);
+
+#ifdef _WIN32
+
+	// Convert microseconds to seconds for SYSTEMTIME
+	int64_t seconds_since_epoch = utc_micros / 1000000;
+
+	// Convert seconds to FILETIME
+	FILETIME ft_utc;
+	ULARGE_INTEGER utc_time;
+	// Add Windows epoch offset
+	utc_time.QuadPart = static_cast<uint64_t>(seconds_since_epoch) * 10000000 + 11644473600LL * 10000000;
+	ft_utc.dwLowDateTime = utc_time.LowPart;
+	ft_utc.dwHighDateTime = utc_time.HighPart;
+
+	// Convert FILETIME to SYSTEMTIME
+	SYSTEMTIME utc_system_time;
+	auto res_fttt = FileTimeToSystemTime(&ft_utc, &utc_system_time);
+	if (res_fttt == 0) {
+		std::string msg = "FileTimeToSystemTime failed, input value: " + std::to_string(utc_micros) +
+		                  ", error code: " + std::to_string(GetLastError());
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Get the default time zone information
+	TIME_ZONE_INFORMATION tz_info;
+	DWORD res_tzinfo = GetTimeZoneInformation(&tz_info);
+	if (res_tzinfo == TIME_ZONE_ID_INVALID) {
+		std::string msg = "GetTimeZoneInformation failed, input value: " + std::to_string(utc_micros) +
+		                  ", error code: " + std::to_string(GetLastError());
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Convert UTC SYSTEMTIME to local SYSTEMTIME
+	SYSTEMTIME local_system_time;
+	auto res_tzspec = SystemTimeToTzSpecificLocalTime(&tz_info, &utc_system_time, &local_system_time);
+	if (res_tzspec == 0) {
+		std::string msg = "SystemTimeToTzSpecificLocalTime failed, input value: " + std::to_string(utc_micros) +
+		                  ", error code: " + std::to_string(GetLastError());
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Convert local SYSTEMTIME back to FILETIME
+	FILETIME ft_local;
+	auto res_sttft = SystemTimeToFileTime(&local_system_time, &ft_local);
+	if (res_sttft == 0) {
+		std::string msg = "SystemTimeToFileTime failed, input value: " + std::to_string(utc_micros) +
+		                  ", error code: " + std::to_string(GetLastError());
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Convert both FILETIMEs to ULARGE_INTEGER for arithmetic
+	ULARGE_INTEGER local_time;
+	local_time.LowPart = ft_local.dwLowDateTime;
+	local_time.HighPart = ft_local.dwHighDateTime;
+
+	// Calculate the offset in microseconds
+	return static_cast<int64_t>(local_time.QuadPart - utc_time.QuadPart) / 10;
+
+#else //!_WiN32
+
+	// Convert microseconds to seconds
+	time_t utc_seconds_since_epoch = utc_micros / 1000000;
+
+	// Break down UTC time into a tm struct
+	struct tm utc_time_struct;
+	// Thread-safe UTC conversion
+	auto res_gmt = gmtime_r(&utc_seconds_since_epoch, &utc_time_struct);
+	if (res_gmt == nullptr) {
+		std::string msg =
+		    "gmtime_r failed, input value: " + std::to_string(utc_micros) + ", error code: " + std::to_string(errno);
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Convert UTC time to local time in the default time zone
+	struct tm local_time_struct;
+	// Thread-safe local time conversion
+	auto res_local = localtime_r(&utc_seconds_since_epoch, &local_time_struct);
+	if (res_local == nullptr) {
+		std::string msg =
+		    "localtime_r failed, input value: " + std::to_string(utc_micros) + ", error code: " + std::to_string(errno);
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+	// DST confuses mktime
+	local_time_struct.tm_isdst = 0;
+
+	// Convert broken-down times to time_t (seconds since epoch)
+	time_t local_time = mktime(&local_time_struct);
+	if (local_time == -1) {
+		std::string msg = "mktime local failed, input value: " + std::to_string(utc_micros) +
+		                  ", error code: " + std::to_string(errno);
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+	time_t utc_time = mktime(&utc_time_struct);
+	if (utc_time == -1) {
+		std::string msg =
+		    "mktime utc failed, input value: " + std::to_string(utc_micros) + ", error code: " + std::to_string(errno);
+		duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "timezone", msg, duckdb::SQLStateType::ST_HY000,
+		                            hstmt->dbc->GetDataSourceName());
+		return 0;
+	}
+
+	// Calculate the offset in seconds
+	int64_t offset_seconds = static_cast<int64_t>(difftime(local_time, utc_time));
+
+	// Convert offset to microseconds
+	return offset_seconds * 1000000;
+
+#endif // _WIN32
+}

src/statement/statement_functions.cpp

@@ -332,6 +332,11 @@ SQLRETURN duckdb::GetDataStmtResult(OdbcHandleStmt *hstmt, SQLUSMALLINT col_or_p
 		*str_len_or_ind_ptr = SQL_NULL_DATA;
 		return SQL_SUCCESS;
 	}
+	if (val.type().id() == LogicalType::TIMESTAMP_TZ) {
+		int64_t utc_micros = val.GetValue<int64_t>();
+		int64_t utc_offset_micros = duckdb::OdbcUtils::GetUTCOffsetMicrosFromOS(hstmt, utc_micros);
+		val = Value::TIMESTAMP(timestamp_t(utc_micros + utc_offset_micros));
+	}
 
 	SQLSMALLINT target_type_resolved = target_type;
 	if (target_type_resolved == SQL_C_DEFAULT) {

test/tests/test_timestamp.cpp

@@ -1,5 +1,11 @@
 #include "odbc_test_common.h"
 
+#include <cstdint>
+#include <ctime>
+#include <iomanip>
+#include <mutex>
+#include <sstream>
+
 using namespace odbc_test;
 
 TEST_CASE("Test SQLBindParameter with TIMESTAMP type", "[odbc]") {
@@ -168,3 +174,94 @@ TEST_CASE("Test SQLBindParameter with TIME type", "[odbc]") {
 
 	DISCONNECT_FROM_DATABASE(env, dbc);
 }
+
+static std::mutex &get_static_mutex() {
+	static std::mutex mtx;
+	return mtx;
+}
+
+static int64_t os_utc_offset_seconds(const std::string &timestamp_st) {
+	std::mutex &mtx = get_static_mutex();
+	std::lock_guard<std::mutex> guard(mtx);
+	std::tm input_time = {};
+	std::istringstream ss(timestamp_st);
+	ss >> std::get_time(&input_time, "%Y-%m-%d %H:%M:%S");
+	std::time_t input_instant = std::mktime(&input_time);
+	std::tm local_time = *std::localtime(&input_instant);
+	local_time.tm_isdst = 0;
+	std::tm utc_time = *std::gmtime(&input_instant);
+	std::time_t local_instant = std::mktime(&local_time);
+	std::time_t utc_instant = std::mktime(&utc_time);
+	return static_cast<int64_t>(difftime(local_instant, utc_instant));
+}
+
+static std::tm sql_timestamp_to_tm(const SQL_TIMESTAMP_STRUCT &ts) {
+	std::tm dt = {};
+	dt.tm_year = ts.year - 1900;
+	dt.tm_mon = ts.month - 1;
+	dt.tm_mday = ts.day;
+	dt.tm_hour = ts.hour;
+	dt.tm_min = ts.minute;
+	dt.tm_sec = ts.second;
+	return dt;
+}
+
+static int64_t sql_timestamp_diff_seconds(const SQL_TIMESTAMP_STRUCT &ts1, const SQL_TIMESTAMP_STRUCT &ts2) {
+	std::mutex &mtx = get_static_mutex();
+	std::lock_guard<std::mutex> guard(mtx);
+	std::tm tm1 = sql_timestamp_to_tm(ts1);
+	std::tm tm2 = sql_timestamp_to_tm(ts2);
+	std::time_t instant1 = std::mktime(&tm1);
+	std::time_t instant2 = std::mktime(&tm2);
+	return static_cast<int64_t>(std::difftime(instant1, instant2));
+}
+
+TEST_CASE("Test fetching TIMESTAMP_TZ values as TIMESTAMP", "[odbc]") {
+	SQLHANDLE env;
+	SQLHANDLE dbc;
+
+	HSTMT hstmt = SQL_NULL_HSTMT;
+
+	// Connect to the database using SQLConnect
+	CONNECT_TO_DATABASE(env, dbc);
+
+	// Allocate a statement handle
+	EXECUTE_AND_CHECK("SQLAllocHandle (HSTMT)", hstmt, SQLAllocHandle, SQL_HANDLE_STMT, dbc, &hstmt);
+
+	std::string tm_local_st = "2004-07-19 10:23:54";
+
+	// Process this date as TIMESTAMP_TZ and fetch is as SQL_C_TYPE_TIMESTAMP,
+	// return value must be converted to local using OS-configured time-zone
+	EXECUTE_AND_CHECK("SQLExecDirect", hstmt, SQLExecDirect, hstmt,
+	                  ConvertToSQLCHAR("SELECT '" + tm_local_st + "+00'::TIMESTAMP WITH TIME ZONE"), SQL_NTS);
+	EXECUTE_AND_CHECK("SQLFetch", hstmt, SQLFetch, hstmt);
+	SQL_TIMESTAMP_STRUCT fetched_timestamp_tz;
+	EXECUTE_AND_CHECK("SQLGetData", hstmt, SQLGetData, hstmt, 1, SQL_C_TYPE_TIMESTAMP, &fetched_timestamp_tz,
+	                  sizeof(fetched_timestamp_tz), nullptr);
+
+	// Process the same date as TIMESTAMP, no time-zone conversion performed
+	EXECUTE_AND_CHECK("SQLExecDirect", hstmt, SQLExecDirect, hstmt,
+	                  ConvertToSQLCHAR("SELECT '" + tm_local_st + "'::TIMESTAMP WITHOUT TIME ZONE"), SQL_NTS);
+	EXECUTE_AND_CHECK("SQLFetch", hstmt, SQLFetch, hstmt);
+	SQL_TIMESTAMP_STRUCT fetched_timestamp;
+	EXECUTE_AND_CHECK("SQLGetData", hstmt, SQLGetData, hstmt, 1, SQL_C_TYPE_TIMESTAMP, &fetched_timestamp,
+	                  sizeof(fetched_timestamp), nullptr);
+	REQUIRE(fetched_timestamp.year == 2004);
+	REQUIRE(fetched_timestamp.month == 7);
+	REQUIRE(fetched_timestamp.day == 19);
+	REQUIRE(fetched_timestamp.hour == 10);
+	REQUIRE(fetched_timestamp.minute == 23);
+	REQUIRE(fetched_timestamp.second == 54);
+	REQUIRE(fetched_timestamp.fraction == 0);
+
+	// Get OS time zone offset and compare the fetched results
+	int64_t os_utc_offset = os_utc_offset_seconds(tm_local_st);
+	int64_t fetched_diff = sql_timestamp_diff_seconds(fetched_timestamp_tz, fetched_timestamp);
+	REQUIRE(fetched_diff == os_utc_offset);
+
+	// Free the statement handle
+	EXECUTE_AND_CHECK("SQLFreeStmt (HSTMT)", hstmt, SQLFreeStmt, hstmt, SQL_CLOSE);
+	EXECUTE_AND_CHECK("SQLFreeHandle (HSTMT)", hstmt, SQLFreeHandle, SQL_HANDLE_STMT, hstmt);
+
+	DISCONNECT_FROM_DATABASE(env, dbc);
+}