Merge remote-tracking branch 'origin' into update-to-duckdb-v1.3.0

Author: JelteF

docs/extensions.md

@@ -31,24 +31,32 @@ Installing an extension causes it to be loaded and installed globally for any co
 SELECT duckdb.install_extension('iceberg');
 -- view currently installed extensions
 SELECT * FROM duckdb.extensions;
--- disable or enable an extension
-UPDATE duckdb.extensions SET enabled = (false|true) WHERE name = 'iceberg';
--- remove an extension
-DELETE FROM duckdb.extensions WHERE name = 'iceberg';
+-- Change an extension to stop being automatically loaded in new connections
+SELECT duckdb.auotoload_extension('iceberg', false);
+-- For such extensions, you can still load them manually in a session
+SELECT duckdb.load_extension('iceberg');
 -- You can also install community extensions
 SELECT duckdb.install_extension('prql', 'community');
 ```
 
-There is currently no practical difference between a disabled and uninstalled extension.
-
 ## Supported Extensions
 
 You can install any extension DuckDB extension, but you might run into various issues when trying to use them from Postgres. Often you should be able to work around such issues by using `duckdb.query` or `duckdb.raw_query`. For some extensions pg_duckdb has added dedicated support to Postgres. These extensions are listed below.
 
+### `azure`
+
+Allows reading files from Azure Blob Storage by using `az://...` filepaths.
+
 ### `iceberg`
 
 Iceberg support adds functions to read Iceberg tables and metadata. For a list of iceberg functions, see [pg_duckdb Functions](functions.md).
 
 ### `delta`
 
 Delta support adds the ability to read Delta Lake files via [delta_scan](functions.md#delta_scan).
+
+## Security considerations
+
+By default execution `duckdb.install_extension` and `duckdb.autoload_extension` is only allowed for superusers. This is to prevent users from installing extensions that may have security implications or that may interfere with the database's operation.
+
+That means that users can only use extensions that DuckDB has marked as "auto-installable". If you want to restrict the use of those extensions as well to a specific list of allowed extensions, you can do so by setting the [`duckdb.autoinstall_known_extensions`](settings.md#duckdbautoload_known_extensions) to `false`. This will prevent users from automatically install any known extensions. Note that this requires that any of the extensions you **do** want to allow are already installed by a superuser.

docs/functions.md

@@ -268,8 +268,6 @@ permissions:
 
 ```sql
 GRANT ALL ON FUNCTION duckdb.install_extension(TEXT, TEXT) TO my_admin;
-GRANT ALL ON TABLE duckdb.extensions TO my_admin;
-GRANT ALL ON SEQUENCE duckdb.extensions_table_seq TO my_admin;
 ```
 
 ##### Required Arguments

include/pgduckdb/pg/declarations.hpp

@@ -86,4 +86,6 @@ struct ObjectAddress;
 struct PlanState;
 
 struct Plan;
+
+typedef struct FunctionCallInfoBaseData *FunctionCallInfo;
 }

include/pgduckdb/pg/functions.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "string"
+#include "pgduckdb/pg/declarations.hpp"
+
+namespace pgduckdb::pg {
+
+std::string GetArgString(FunctionCallInfo info, int argno);
+Datum GetArgDatum(FunctionCallInfo info, int argno);
+std::string DatumToString(Datum datum);
+
+} // namespace pgduckdb::pg

include/pgduckdb/pg/relations.hpp

@@ -37,4 +37,8 @@ const char *GetRelationName(Relation rel);
 
 Oid GetOid(Form_pg_class rel);
 
+namespace pg {
+Form_pg_attribute GetAttributeByName(TupleDesc tupdesc, const char *colname);
+}
+
 } // namespace pgduckdb

include/pgduckdb/pgduckdb_duckdb.hpp

@@ -65,6 +65,7 @@ class DuckDBManager {
 	void LoadSecrets(duckdb::ClientContext &);
 	void DropSecrets(duckdb::ClientContext &);
 	void LoadExtensions(duckdb::ClientContext &);
+	void InstallExtensions(duckdb::ClientContext &);
 	void LoadFunctions(duckdb::ClientContext &);
 	void RefreshConnectionState(duckdb::ClientContext &);
 

include/pgduckdb/pgduckdb_extensions.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace pgduckdb {
+
+/* constants for duckdb.extensions */
+#define Natts_duckdb_extension           3
+#define Anum_duckdb_extension_name       1
+#define Anum_duckdb_extension_autoload   2
+#define Anum_duckdb_extension_repository 3
+
+typedef struct DuckdbExension {
+	std::string name;
+	bool autoload;
+	std::string repository;
+} DuckdbExtension;
+
+extern std::vector<DuckdbExtension> ReadDuckdbExtensions();
+
+namespace ddb {
+
+extern std::string LoadExtensionQuery(const std::string &extension_name);
+extern std::string InstallExtensionQuery(const std::string &extension_name, const std::string &repository);
+
+} // namespace ddb
+} // namespace pgduckdb

include/pgduckdb/pgduckdb_options.hpp

@@ -6,15 +6,18 @@
 namespace pgduckdb {
 
 /* constants for duckdb.extensions */
-#define Natts_duckdb_extension       2
-#define Anum_duckdb_extension_name   1
-#define Anum_duckdb_extension_enable 2
+#define Natts_duckdb_extension           3
+#define Anum_duckdb_extension_name       1
+#define Anum_duckdb_extension_autoload   2
+#define Anum_duckdb_extension_repository 3
 
 typedef struct DuckdbExension {
 	std::string name;
-	bool enabled;
+	bool autoload;
+	std::string repository;
 } DuckdbExtension;
 
 extern std::vector<DuckdbExtension> ReadDuckdbExtensions();
+extern std::string DuckdbInstallExtensionQuery(const std::string &extension_name, const std::string &repository);
 
 } // namespace pgduckdb

sql/pg_duckdb--0.3.0--1.0.0.sql

@@ -64,10 +64,27 @@ LANGUAGE C;
 
 DROP FUNCTION duckdb.install_extension(TEXT);
 CREATE FUNCTION duckdb.install_extension(extension_name TEXT, source TEXT DEFAULT 'core') RETURNS void
+    SECURITY DEFINER
     SET search_path = pg_catalog, pg_temp
+    SET duckdb.force_execution = false
     LANGUAGE C AS 'MODULE_PATHNAME', 'install_extension';
 REVOKE ALL ON FUNCTION duckdb.install_extension(TEXT, TEXT) FROM PUBLIC;
 
+CREATE FUNCTION duckdb.autoload_extension(extension_name TEXT, autoload BOOLEAN DEFAULT TRUE) RETURNS void
+    SECURITY DEFINER
+    SET search_path = pg_catalog, pg_temp
+    SET duckdb.force_execution = false
+    LANGUAGE C AS 'MODULE_PATHNAME', 'duckdb_autoload_extension';
+REVOKE ALL ON FUNCTION duckdb.autoload_extension(TEXT, BOOLEAN) FROM PUBLIC;
+
+CREATE FUNCTION duckdb.load_extension(extension_name TEXT) RETURNS void
+    SET search_path = pg_catalog, pg_temp
+    SET duckdb.force_execution = false
+    LANGUAGE C AS 'MODULE_PATHNAME', 'duckdb_load_extension';
+-- load_extension is allowed for all users, because it is trivial for users run
+-- a raw query that does the same. This function only exists for completeness
+-- and convenience.
+
 -- The min aggregate was somehow missing from the list of aggregates in 0.3.0
 CREATE AGGREGATE @extschema@.min(duckdb.unresolved_type) (
     SFUNC = duckdb_unresolved_type_state_trans,
@@ -725,4 +742,6 @@ RETURNS TEXT
 SET search_path = pg_catalog, pg_temp
 LANGUAGE C AS 'MODULE_PATHNAME', 'pgduckdb_create_azure_secret';
 
-
+ALTER TABLE duckdb.extensions ADD COLUMN repository TEXT NOT NULL DEFAULT 'core';
+ALTER TABLE duckdb.extensions RENAME COLUMN enabled TO autoload;
+ALTER TABLE duckdb.extensions ALTER COLUMN autoload SET NOT NULL;

src/pg/functions.cpp

@@ -0,0 +1,52 @@
+#include "pgduckdb/pg/functions.hpp"
+#include "duckdb/common/exception.hpp"
+
+#include "pgduckdb/pgduckdb_utils.hpp"
+
+extern "C" {
+#include "postgres.h"
+#include "fmgr.h"
+#include "utils/builtins.h"
+}
+
+namespace pgduckdb::pg {
+
+std::string
+GetArgString(PG_FUNCTION_ARGS, int argno) {
+	if (PG_NARGS() <= argno) {
+		throw duckdb::InvalidInputException("argument %d is required", argno);
+	}
+
+	if (PG_ARGISNULL(argno)) {
+		throw duckdb::InvalidInputException("argument %d cannot be NULL", argno);
+	}
+
+	return DatumToString(PG_GETARG_DATUM(argno));
+}
+
+Datum
+GetArgDatum(PG_FUNCTION_ARGS, int argno) {
+	if (PG_NARGS() <= argno) {
+		throw duckdb::InvalidInputException("argument %d is required", argno);
+	}
+
+	if (PG_ARGISNULL(argno)) {
+		throw duckdb::InvalidInputException("argument %d cannot be NULL", argno);
+	}
+
+	return PG_GETARG_DATUM(argno);
+}
+
+namespace {
+char *
+DatumToCstring(Datum datum) {
+	return text_to_cstring(DatumGetTextPP(datum));
+}
+} // namespace
+
+std::string
+DatumToString(Datum datum) {
+	return std::string(PostgresFunctionGuard(DatumToCstring, datum));
+}
+
+} // namespace pgduckdb::pg

src/pg/relations.cpp

@@ -177,4 +177,19 @@ GetOid(Form_pg_class rel) {
 	return rel->oid;
 }
 
+namespace pg {
+
+Form_pg_attribute
+GetAttributeByName(TupleDesc tupdesc, const char *colname) {
+	for (int i = 0; i < tupdesc->natts; i++) {
+		Form_pg_attribute attr = TupleDescAttr(tupdesc, i);
+		if (strcmp(NameStr(attr->attname), colname) == 0) {
+			return attr;
+		}
+	}
+	return NULL; // Return NULL if the column name is not found
+}
+
+} // namespace pg
+
 } // namespace pgduckdb

src/pgduckdb_duckdb.cpp

@@ -15,7 +15,7 @@
 #include "pgduckdb/pgduckdb_fdw.hpp"
 #include "pgduckdb/pgduckdb_guc.hpp"
 #include "pgduckdb/pgduckdb_metadata_cache.hpp"
-#include "pgduckdb/pgduckdb_options.hpp"
+#include "pgduckdb/pgduckdb_extensions.hpp"
 #include "pgduckdb/pgduckdb_secrets_helper.hpp"
 #include "pgduckdb/pgduckdb_userdata_cache.hpp"
 #include "pgduckdb/pgduckdb_utils.hpp"
@@ -31,7 +31,7 @@ extern "C" {
 #include "catalog/namespace.h"
 #include "common/file_perm.h"
 #include "lib/stringinfo.h"
-#include "miscadmin.h" // superuser
+#include "miscadmin.h"        // superuser
 #include "nodes/value.h"      // strVal
 #include "utils/fmgrprotos.h" // pg_sequence_last_value
 #include "utils/lsyscache.h"  // get_relname_relid
@@ -189,6 +189,9 @@ DuckDBManager::Initialize() {
 	}
 
 	LoadFunctions(context);
+	if (duckdb_autoinstall_known_extensions) {
+		InstallExtensions(context);
+	}
 	LoadExtensions(context);
 }
 
@@ -239,12 +242,21 @@ DuckDBManager::LoadExtensions(duckdb::ClientContext &context) {
 	auto duckdb_extensions = ReadDuckdbExtensions();
 
 	for (auto &extension : duckdb_extensions) {
-		if (extension.enabled) {
-			DuckDBQueryOrThrow(context, "LOAD " + extension.name);
+		if (extension.autoload) {
+			DuckDBQueryOrThrow(context, ddb::LoadExtensionQuery(extension.name));
 		}
 	}
 }
 
+void
+DuckDBManager::InstallExtensions(duckdb::ClientContext &context) {
+	auto duckdb_extensions = ReadDuckdbExtensions();
+
+	for (auto &extension : duckdb_extensions) {
+		DuckDBQueryOrThrow(context, ddb::InstallExtensionQuery(extension.name, extension.repository));
+	}
+}
+
 void
 DuckDBManager::RefreshConnectionState(duckdb::ClientContext &context) {
 	const auto extensions_table_last_seq = GetSeqLastValue("extensions_table_seq");
@@ -328,8 +340,8 @@ DuckDBManager::GetConnection(bool force_transaction) {
  * Returns the cached connection to the global DuckDB instance, but does not do
  * any checks required to correctly initialize the DuckDB transaction nor
  * refreshes the secrets/extensions/etc. Only use this in rare cases where you
- * know for sure that the connection is already initialized for the correctly
- * for the current query, and you just want a pointer to it.
+ * know for sure that the connection is already initialized correctly for the
+ * current query, and you just want a pointer to it.
  */
 duckdb::Connection *
 DuckDBManager::GetConnectionUnsafe() {