Bump duckdb to v1.5.1

.gitignore

@@ -42,3 +42,4 @@ pg_lake_iceberg/logs/*
 pg_lake_table/logs/
 .volume/
 Dockerfile.alpine
+duckdb_pglake/.cache/

Pipfile

@@ -19,7 +19,7 @@ pyspark = {extras = ["sql"], version = "==3.5.3"}
 pytest-postgresql = "~=7.0"
 psycopg-binary = "~=3.2"
 sqlalchemy = "~=2.0"
-duckdb = "==1.4.3"
+duckdb = "==1.5.1"
 
 [dev-packages]
 black = "==25.9.0"

duckdb_pglake/Makefile

@@ -11,7 +11,7 @@ EXT_FLAGS=-DBUILD_EXTENSIONS="tpch;json;icu"
 # GitHub actions uses a hash of the Makefile and source
 # files as a cache key. When we update the DuckDB version,
 # we should update the line below to trigger a new hash.
-DUCKDB_VERSION=1.4.4
+DUCKDB_VERSION=1.5.1
 
 # We install libduckdb.so into the postgres direcotry
 PG_CONFIG ?= pg_config

duckdb_pglake/extension_config.cmake

@@ -1,19 +1,19 @@
 # External extensions to link into libduckdb
 duckdb_extension_load(httpfs
     GIT_URL https://github.com/duckdb/duckdb-httpfs
-    GIT_TAG 13f8a814d41a978c3f19eb1dc76069489652ea6f
+    GIT_TAG 7e86e7a5e5a1f01f458361bebdfa9b0a9a73a619
     INCLUDE_DIR src/include
     ADD_PATCHES
 )
 
 duckdb_extension_load(aws
     GIT_URL https://github.com/duckdb/duckdb-aws
-    GIT_TAG bc15d211f282d1d78fc0d9fda3d09957ba776423
+    GIT_TAG b2649e68341a9ee717588dd23f277904727ce793
 )
 
 duckdb_extension_load(azure
     GIT_URL https://github.com/duckdb/duckdb-azure
-    GIT_TAG 7e1ac3333d946a6bf5b4552722743e03f30a47cd
+    GIT_TAG ea6ffae3710ec568ce08579dbfc0cddc8c759227
 )
 
 # Extension from this repo

duckdb_pglake/patches/duckdb-postgres/composite-type-resolution.patch

@@ -1,23 +1,23 @@
 diff --git a/src/include/postgres_utils.hpp b/src/include/postgres_utils.hpp
-index ea04300..38672a9 100644
+index 40f1516..d3d6b69 100644
 --- a/src/include/postgres_utils.hpp
 +++ b/src/include/postgres_utils.hpp
-@@ -13,6 +13,7 @@
+@@ -13,12 +13,14 @@
  #include "postgres_version.hpp"
 
  namespace duckdb {
 +class PostgresConnection;
  class PostgresSchemaEntry;
  class PostgresTransaction;
 
-@@ -21,5 +22,6 @@ struct PostgresTypeData {
+ struct PostgresTypeData {
  	int64_t type_modifier = 0;
  	string type_name;
 +	int64_t type_oid = 0;
  	idx_t array_dimensions = 0;
  };
 
-@@ -64,7 +66,8 @@ public:
+@@ -63,7 +65,8 @@ public:
  	static LogicalType ToPostgresType(const LogicalType &input);
  	static LogicalType TypeToLogicalType(optional_ptr<PostgresTransaction> transaction,
  	                                     optional_ptr<PostgresSchemaEntry> schema, const PostgresTypeData &input,
@@ -28,11 +28,11 @@ index ea04300..38672a9 100644
  	static string PostgresOidToName(uint32_t oid);
  	static uint32_t ToPostgresOid(const LogicalType &input);
 diff --git a/src/include/storage/postgres_table_set.hpp b/src/include/storage/postgres_table_set.hpp
-index 422c2a1..997cf71 100644
+index fcdff54..b5ad911 100644
 --- a/src/include/storage/postgres_table_set.hpp
 +++ b/src/include/storage/postgres_table_set.hpp
 @@ -46,11 +46,13 @@ protected:
- 	void AlterTable(PostgresTransaction &transaction, RemoveColumnInfo &info);
+ 	void AlterTable(ClientContext &context, PostgresTransaction &transaction, RemoveColumnInfo &info);
 
  	static void AddColumn(optional_ptr<PostgresTransaction> transaction, optional_ptr<PostgresSchemaEntry> schema,
 -	                      PostgresResult &result, idx_t row, PostgresTableInfo &table_info);
@@ -48,7 +48,7 @@ index 422c2a1..997cf71 100644
  	void CreateEntries(PostgresTransaction &transaction, PostgresResult &result, idx_t start, idx_t end);
 
 diff --git a/src/postgres_utils.cpp b/src/postgres_utils.cpp
-index 2d9f5d9..6f2a3f2 100644
+index 646696b..b9ad53b 100644
 --- a/src/postgres_utils.cpp
 +++ b/src/postgres_utils.cpp
 @@ -1,4 +1,5 @@
@@ -57,7 +57,7 @@ index 2d9f5d9..6f2a3f2 100644
  #include "storage/postgres_schema_entry.hpp"
  #include "storage/postgres_transaction.hpp"
  #include "postgres_type_oids.hpp"
-@@ -85,7 +86,8 @@ LogicalType PostgresUtils::RemoveAlias(const LogicalType &type) {
+@@ -74,7 +75,8 @@ LogicalType PostgresUtils::RemoveAlias(const LogicalType &type) {
 
  LogicalType PostgresUtils::TypeToLogicalType(optional_ptr<PostgresTransaction> transaction,
                                               optional_ptr<PostgresSchemaEntry> schema,
@@ -67,12 +67,13 @@ index 2d9f5d9..6f2a3f2 100644
  	auto &pgtypename = type_info.type_name;
 
  	// postgres array types start with an _
-@@ -113,7 +115,14 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr<PostgresTransaction> t
+@@ -101,8 +103,15 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr<PostgresTransaction> t
+ 		PostgresTypeData child_type_info;
  		child_type_info.type_name = pgtypename.substr(1);
  		child_type_info.type_modifier = type_info.type_modifier;
 +		if (connection) {
 +			D_ASSERT(type_info.type_oid != 0);
-+			auto elem_result = connection->Query(StringUtil::Format(
++			auto elem_result = connection->Query(nullptr, StringUtil::Format(
 +			    "SELECT typelem FROM pg_type WHERE oid = %d", type_info.type_oid));
 +			D_ASSERT(elem_result->Count() > 0);
 +			child_type_info.type_oid = elem_result->GetInt64(0, 0);
@@ -83,7 +84,7 @@ index 2d9f5d9..6f2a3f2 100644
  		// construct the child type based on the number of dimensions
  		for (idx_t i = 1; i < dimensions; i++) {
  			PostgresType new_pg_type;
-@@ -201,7 +203,52 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr<PostgresTransaction> t
+@@ -190,7 +199,52 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr<PostgresTransaction> t
  		return LogicalType::LIST(LogicalType::DOUBLE);
  	} else {
  		if (!transaction) {
@@ -100,7 +101,7 @@ index 2d9f5d9..6f2a3f2 100644
 +				    "AND a.attnum > 0 AND NOT a.attisdropped "
 +				    "ORDER BY a.attnum",
 +				    type_info.type_oid);
-+				auto result = connection->Query(query);
++				auto result = connection->Query(nullptr, query);
 +				auto rows = result->Count();
 +				if (rows > 0) {
 +					child_list_t<LogicalType> child_types;
@@ -124,7 +125,7 @@ index 2d9f5d9..6f2a3f2 100644
 +				    "JOIN pg_type bt ON bt.oid = t.typbasetype "
 +				    "WHERE t.oid = %d AND t.typtype = 'd'",
 +				    type_info.type_oid);
-+				auto domain_result = connection->Query(domain_query);
++				auto domain_result = connection->Query(nullptr, domain_query);
 +				if (domain_result->Count() > 0) {
 +					PostgresTypeData base_type_info;
 +					base_type_info.type_name = domain_result->GetString(0, 0);
@@ -138,19 +139,19 @@ index 2d9f5d9..6f2a3f2 100644
  			return LogicalType::VARCHAR;
  		}
 diff --git a/src/storage/postgres_table_set.cpp b/src/storage/postgres_table_set.cpp
-index efe37a1..07d3ee6 100644
+index ae53dc6..dc42213 100644
 --- a/src/storage/postgres_table_set.cpp
 +++ b/src/storage/postgres_table_set.cpp
-@@ -24,16 +24,16 @@ SELECT pg_namespace.oid AS namespace_id, relname, relpages, attname,
--    pg_type.typname type_name, atttypmod type_modifier, pg_attribute.attndims ndim,
+@@ -23,7 +23,7 @@ string PostgresTableSet::GetInitializeQuery(const string &schema, const string &
+ 	string base_query = R"(
+ SELECT pg_namespace.oid AS namespace_id, relname, relpages, attname,
+     pg_type.typname type_name, atttypmod type_modifier, pg_attribute.attndims ndim,
 -    attnum, pg_attribute.attnotnull AS notnull, NULL constraint_id,
-+    pg_type.typname type_name, atttypmod type_modifier, pg_attribute.attndims ndim,
 +    atttypid type_oid, attnum, pg_attribute.attnotnull AS notnull, NULL constraint_id,
      NULL constraint_type, NULL constraint_key
  FROM pg_class
  JOIN pg_namespace ON relnamespace = pg_namespace.oid
- JOIN pg_attribute ON pg_class.oid=pg_attribute.attrelid
- JOIN pg_type ON atttypid=pg_type.oid
+@@ -32,7 +32,7 @@ JOIN pg_type ON atttypid=pg_type.oid
  WHERE attnum > 0 AND relkind IN ('r', 'v', 'm', 'f', 'p') ${CONDITION}
  UNION ALL
  SELECT pg_namespace.oid AS namespace_id, relname, NULL relpages, NULL attname, NULL type_name,
@@ -159,8 +160,6 @@ index efe37a1..07d3ee6 100644
      pg_constraint.oid AS constraint_id, contype AS constraint_type,
      conkey AS constraint_key
  FROM pg_class
- JOIN pg_namespace ON relnamespace = pg_namespace.oid
- JOIN pg_constraint ON (pg_class.oid=pg_constraint.conrelid)
 @@ -53,18 +53,19 @@ ORDER BY namespace_id, relname, attnum, constraint_id;
 
  void PostgresTableSet::AddColumn(optional_ptr<PostgresTransaction> transaction,
@@ -173,8 +172,8 @@ index efe37a1..07d3ee6 100644
  	type_info.type_name = result.GetString(row, column_index + 1);
  	type_info.type_modifier = result.GetInt64(row, column_index + 2);
  	type_info.array_dimensions = result.GetInt64(row, column_index + 3);
-+	type_info.type_oid = result.GetInt64(row, column_index + 4);
 -	bool is_not_null = result.GetBool(row, column_index + 5);
++	type_info.type_oid = result.GetInt64(row, column_index + 4);
 +	bool is_not_null = result.GetBool(row, column_index + 6);
  	string default_value;
 
@@ -184,12 +183,16 @@ index efe37a1..07d3ee6 100644
  	table_info.postgres_types.push_back(std::move(postgres_type));
  	table_info.postgres_names.push_back(column_name);
  	ColumnDefinition column(std::move(column_name), std::move(column_type));
-@@ -87,3 +87,3 @@ void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, Postgres
+@@ -84,7 +85,7 @@ void PostgresTableSet::AddColumn(optional_ptr<PostgresTransaction> transaction,
+ }
+
  void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, PostgresTableInfo &table_info) {
 -	idx_t column_index = 9;
 +	idx_t column_index = 10;
  	auto constraint_type = result.GetString(row, column_index + 1);
-@@ -109,12 +109,13 @@ void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, Postgres
+ 	auto constraint_key = result.GetString(row, column_index + 2);
+ 	if (constraint_key.empty() || constraint_key.front() != '{' || constraint_key.back() != '}') {
+@@ -109,12 +110,13 @@ void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, Postgres
 
  void PostgresTableSet::AddColumnOrConstraint(optional_ptr<PostgresTransaction> transaction,
                                               optional_ptr<PostgresSchemaEntry> schema, PostgresResult &result,
@@ -205,7 +208,7 @@ index efe37a1..07d3ee6 100644
  	}
  }
 
-@@ -183,7 +184,7 @@ unique_ptr<PostgresTableInfo> PostgresTableSet::GetTableInfo(PostgresConnection
+@@ -183,7 +185,7 @@ unique_ptr<PostgresTableInfo> PostgresTableSet::GetTableInfo(ClientContext &cont
  	}
  	auto table_info = make_uniq<PostgresTableInfo>(schema_name, table_name);
  	for (idx_t row = 0; row < rows; row++) {

duckdb_pglake/patches/duckdb-postgres/numeric-nan.patch

@@ -1,5 +1,5 @@
 diff --git a/src/include/postgres_binary_reader.hpp b/src/include/postgres_binary_reader.hpp
-index f4bc336..d9e92f1 100644
+index 5b5cca9..cbbc708 100644
 --- a/src/include/postgres_binary_reader.hpp
 +++ b/src/include/postgres_binary_reader.hpp
 @@ -136,9 +136,11 @@ protected:
@@ -29,7 +29,7 @@ index 391429e..ba4162b 100644
 
  struct PostgresConversion {
 diff --git a/src/postgres_binary_reader.cpp b/src/postgres_binary_reader.cpp
-index 86ebe30..9a49f8c 100644
+index c7fd72e..dd96b11 100644
 --- a/src/postgres_binary_reader.cpp
 +++ b/src/postgres_binary_reader.cpp
 @@ -137,6 +137,7 @@ PostgresDecimalConfig PostgresBinaryReader::ReadDecimalConfig() {
@@ -55,7 +55,7 @@ index 86ebe30..9a49f8c 100644
  			break;
  		}
  		D_ASSERT(value_len == sizeof(double));
-@@ -284,22 +288,26 @@ void PostgresBinaryReader::ReadValue(const LogicalType &type, const PostgresType
+@@ -294,22 +298,26 @@ void PostgresBinaryReader::ReadValue(const LogicalType &type, const PostgresType
  		if (value_len < sizeof(uint16_t) * 4) {
  			throw InvalidInputException("Need at least 8 bytes to read a Postgres decimal. Got %d", value_len);
  		}
@@ -87,12 +87,12 @@ index 86ebe30..9a49f8c 100644
  	}
 
 diff --git a/src/postgres_text_reader.cpp b/src/postgres_text_reader.cpp
-index 066e072..e3cfa67 100644
+index 3f41a8f..c1e3632 100644
 --- a/src/postgres_text_reader.cpp
 +++ b/src/postgres_text_reader.cpp
-@@ -321,6 +321,19 @@ void PostgresTextReader::ConvertVector(Vector &source, Vector &target, const Pos
- 	case LogicalTypeId::BLOB:
- 		ConvertBlob(source, target, count);
+@@ -370,6 +370,19 @@ void PostgresTextReader::ConvertVector(Vector &source, Vector &target, const Pos
+ 	case LogicalTypeId::GEOMETRY:
+ 		ConvertGeometry(source, target, count);
  		break;
 +	case LogicalTypeId::DECIMAL: {
 +		auto strings = FlatVector::GetData<string_t>(source);

duckdb_pglake/patches/duckdb-postgres/snapshot.patch

@@ -1,8 +1,8 @@
 diff --git a/src/include/postgres_scanner.hpp b/src/include/postgres_scanner.hpp
-index 0f9cda7..9f93468 100644
+index 3e0162c..7c8fa19 100644
 --- a/src/include/postgres_scanner.hpp
 +++ b/src/include/postgres_scanner.hpp
-@@ -48,6 +48,8 @@ public:
+@@ -50,6 +50,8 @@ public:
  	bool use_text_protocol = false;
  	idx_t max_threads = 1;
 
@@ -12,34 +12,33 @@ index 0f9cda7..9f93468 100644
  	void SetTablePages(idx_t approx_num_pages);
 
 diff --git a/src/postgres_query.cpp b/src/postgres_query.cpp
-index 700db2f..c7c5101 100644
+index f999617..771427c 100644
 --- a/src/postgres_query.cpp
 +++ b/src/postgres_query.cpp
-@@ -43,6 +43,9 @@ static unique_ptr<FunctionData> PGQueryBind(ClientContext &context, TableFunctio
+@@ -44,6 +44,9 @@ static unique_ptr<FunctionData> PGQueryBind(ClientContext &context, TableFunctio
  		if (kv.first == "use_transaction") {
  			use_transaction = BooleanValue::Get(kv.second);
  		}
 +		if (kv.first == "snapshot") {
 +			result->snapshot = kv.second.GetValue<string>();
 +		}
  	}
- 	result->use_transaction = use_transaction;
 
-@@ -97,6 +100,8 @@ static unique_ptr<FunctionData> PGQueryBind(ClientContext &context, TableFunctio
- PostgresQueryFunction::PostgresQueryFunction()
+ 	vector<Value> param_values;
+@@ -123,6 +126,7 @@ PostgresQueryFunction::PostgresQueryFunction()
      : TableFunction("postgres_query", {LogicalType::VARCHAR, LogicalType::VARCHAR}, nullptr, PGQueryBind) {
  	named_parameters["use_transaction"] = LogicalType::BOOLEAN;
+ 	named_parameters["params"] = LogicalType::ANY;
 +	named_parameters["snapshot"] = LogicalType::VARCHAR;
-+
  	PostgresScanFunction scan_function;
  	init_global = scan_function.init_global;
  	init_local = scan_function.init_local;
 diff --git a/src/postgres_scanner.cpp b/src/postgres_scanner.cpp
-index bb4136f..2530172 100644
+index eb4360a..3de7dbe 100644
 --- a/src/postgres_scanner.cpp
 +++ b/src/postgres_scanner.cpp
 @@ -67,9 +67,10 @@ private:
- static void PostgresGetSnapshot(PostgresVersion version, const PostgresBindData &bind_data,
+ static void PostgresGetSnapshot(ClientContext &context, PostgresVersion version, const PostgresBindData &bind_data,
                                  PostgresGlobalState &gstate) {
  	unique_ptr<PostgresResult> result;
 -	// by default disable snapshotting
@@ -52,8 +51,8 @@ index bb4136f..2530172 100644
  		return;
  	}
  	if (version.type_v == PostgresInstanceType::AURORA) {
-@@ -93,7 +94,6 @@ static void PostgresGetSnapshot(PostgresVersion version, const PostgresBindData
- 	    con.TryQuery("SELECT pg_is_in_recovery(), pg_export_snapshot(), (select count(*) from pg_stat_wal_receiver)");
+@@ -93,7 +94,6 @@ static void PostgresGetSnapshot(ClientContext &context, PostgresVersion version,
+ 	    context, "SELECT pg_is_in_recovery(), pg_export_snapshot(), (select count(*) from pg_stat_wal_receiver)");
  	if (result) {
  		auto in_recovery = result->GetBool(0, 0) || result->GetInt64(0, 2) > 0;
 -		gstate.snapshot = "";
@@ -74,7 +73,7 @@ index bb4136f..2530172 100644
 +	}
 +
  	auto con = PostgresConnection::Open(bind_data->dsn, bind_data->attach_path);
- 	auto version = con.GetPostgresVersion();
+ 	auto version = con.GetPostgresVersion(context);
  	// query the table schema so we can interpret the bits in the pages
 @@ -315,6 +324,9 @@ static unique_ptr<GlobalTableFunctionState> PostgresInitGlobalState(ClientContex
  	auto &bind_data = input.bind_data->Cast<PostgresBindData>();
@@ -90,8 +89,8 @@ index bb4136f..2530172 100644
  	} else {
  		auto con = PostgresConnection::Open(bind_data.dsn, bind_data.attach_path);
  		if (bind_data.use_transaction) {
--			PostgresScanConnect(con, string());
-+			PostgresScanConnect(con, result->snapshot);
+-			PostgresScanConnect(context, con, string());
++			PostgresScanConnect(context, con, result->snapshot);
  		}
  		result->SetConnection(std::move(con));
  	}

duckdb_pglake/patches/duckdb/parquet-virtual-column-stats.patch

@@ -0,0 +1,27 @@
+diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp
+--- a/extension/parquet/parquet_reader.cpp
++++ b/extension/parquet/parquet_reader.cpp
+@@ -1286,7 +1286,10 @@
+ 	unique_ptr<BaseStatistics> GetColumnStatistics(const StorageIndex &storage_index) override {
+ 		const idx_t primary_index = storage_index.GetPrimaryIndex();
+ 		D_ASSERT(metadata.row_groups.size() > row_group_idx);
+-		D_ASSERT(root_schema->children.size() > primary_index);
++
++		if (primary_index >= root_schema->children.size()) {
++			return nullptr;
++		}
+
+ 		const auto &row_group = metadata.row_groups[row_group_idx];
+ 		const auto &column_schema = root_schema->children[primary_index];
+@@ -1296,7 +1299,10 @@
+ 	bool MinMaxIsExact(const BaseStatistics &, const StorageIndex &storage_index) override {
+ 		const idx_t primary_index = storage_index.GetPrimaryIndex();
+ 		D_ASSERT(metadata.row_groups.size() > row_group_idx);
+-		D_ASSERT(root_schema->children.size() > primary_index);
++
++		if (primary_index >= root_schema->children.size()) {
++			return false;
++		}
+
+ 		const auto &row_group = metadata.row_groups[row_group_idx];
+ 		const auto &column_chunk = row_group.columns[primary_index];