Merge remote-tracking branch 'origin/main' into empty_stubs_improvements

# Conflicts:
#	test/CMakeLists.txt

Author: maiadegraaf

.github/workflows/ODBC.yml

@@ -351,7 +351,7 @@ jobs:
 
   odbc-merge-vendoring-pr:
     name: Merge vendoring PR 
-    if: ${{ github.repository == 'duckdb/duckdb-odbc' && github.event_name == 'pull_request' && github.head_ref == format('vendoring-{0}', github.ref_name) }}
+    if: ${{ github.repository == 'duckdb/duckdb-odbc' && github.event_name == 'pull_request' && github.head_ref == format('vendoring-{0}', github.base_ref) }}
     needs:
       - odbc-linux-amd64
       - odbc-linux-aarch64
@@ -368,8 +368,8 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-            # echo "Merging PR number: ${{ github.event.pull_request.number }} with message: ${{ github.event.pull_request.title }}"
-            gh pr merge vendoring-${{ github.ref_name }} \
+            echo "Merging PR number: ${{ github.event.pull_request.number }} with message: ${{ github.event.pull_request.title }}"
+            gh pr merge vendoring-${{ github.base_ref }} \
              --rebase \
              --subject "${{ github.event.pull_request.title }}" \
              --body ""
@@ -378,9 +378,9 @@ jobs:
         id: update_vendoring_branch
         if: ${{ steps.merge_vendoring_pr.outcome == 'success' }}
         run: |
-            # Delete vendoring-${{ github.ref_name }} branch and re-create it for future PRs
-            git push --delete origin vendoring-${{ github.ref_name }}
+            # Delete vendoring-${{ github.base_ref }} branch and re-create it for future PRs
+            git push --delete origin vendoring-${{ github.base_ref }}
             git checkout --track origin/main
             git pull --ff-only
-            git branch vendoring-${{ github.ref_name }}
-            git push origin vendoring-${{ github.ref_name }}
+            git branch vendoring-${{ github.base_ref }}
+            git push origin vendoring-${{ github.base_ref }}

.github/workflows/Vendor.yml

@@ -51,7 +51,6 @@ jobs:
           git checkout ${{ inputs.duckdb-sha }}
 
       - name: Vendor sources
-        if: ${{ steps.checkout_engine_rev.outcome == 'success' }}
         id: vendor
         run: |
           REV=$(cd .git/duckdb && git rev-parse --short HEAD && cd ../..)
@@ -73,7 +72,6 @@ jobs:
 
       - name: Commit and push the changes
         id: commit_and_push
-        if: ${{ steps.vendor.outcome == 'success' }}
         run: |
           MSG="Update vendored DuckDB sources to ${{ steps.vendor.outputs.vendor_rev }}"
           git commit -m "${MSG}"
@@ -89,7 +87,6 @@ jobs:
 
       - name: Check PR exists
         id: check_pr_exists
-        if: ${{ steps.commit_and_push.outcome == 'success' }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -105,7 +102,6 @@ jobs:
 
       - name: Prepare PR message
         id: prepare_pr_message
-        if: ${{ steps.check_pr_exists.outcome == 'success' }}
         run: |
             DATE="$(date +"%Y-%m-%d %H:%M:%S")"
             CHANGE_LABEL="duckdb/duckdb#${{ steps.vendor.outputs.vendor_rev }}"
@@ -116,7 +112,7 @@ jobs:
 
       - name: Create PR
         id: create_pr
-        if: ${{ steps.prepare_pr_message.outcome == 'success' && steps.check_pr_exists.outputs.pr_exists == 'false' }}
+        if: ${{ steps.check_pr_exists.outputs.pr_exists == 'false' }}
         env:
           # We cannot use default workflow's GITHUB_TOKEN here, because
           # it is restricted to not trigger 'pull_request' event that
@@ -136,7 +132,7 @@ jobs:
 
       - name: Update PR
         id: update_pr
-        if: ${{ steps.prepare_pr_message.outcome == 'success' && steps.check_pr_exists.outputs.pr_exists == 'true' }}
+        if: ${{ steps.check_pr_exists.outputs.pr_exists == 'true' }}
         env:
           # We cannot use default workflow's GITHUB_TOKEN here, because
           # it is restricted to not trigger 'pull_request' event that

CMakeLists.txt

@@ -1,5 +1,6 @@
 #include "connect.hpp"
 #include "duckdb/main/db_instance_cache.hpp"
+#include "duckdb/common/virtual_file_system.hpp"
 #include <iostream>
 #include <utility>
 
@@ -129,6 +130,18 @@ SQLRETURN Connect::ReadFromIniFile() {
 	return SQL_SUCCESS;
 }
 
+static void NormalizeWindowsPathSeparators(case_insensitive_map_t<duckdb::Value> &config_map,
+                                           const std::string &option_name) {
+	auto it = config_map.find(option_name);
+	if (it == config_map.end()) {
+		return;
+	}
+	auto value_str = it->second.GetValue<std::string>();
+	auto value_str_norm = duckdb::StringUtil::Replace(value_str, "\\", "/");
+	config_map.erase(option_name);
+	config_map.emplace(option_name, std::move(value_str_norm));
+}
+
 SQLRETURN Connect::SetConnection() {
 #if defined ODBC_LINK_ODBCINST || defined WIN32
 	ReadFromIniFile();
@@ -141,8 +154,30 @@ SQLRETURN Connect::SetConnection() {
 	config_map.erase("dsn");
 
 	config.SetOptionByName("duckdb_api", "odbc");
+
+	// When 'enable_external_access' is set to 'false' then it is not allowed to change
+	// 'allowed_paths' or 'allowed_directories' options, so we are setting 'enable_external_access'
+	// after all other options
+	std::string enable_external_access;
+	auto it = config_map.find("enable_external_access");
+	if (it != config_map.end()) {
+		enable_external_access = it->second.GetValue<std::string>();
+		config_map.erase(it);
+	}
+
+#ifdef _WIN32
+	// Back slashes are not accepted for 'allowed_paths' or 'allowed_directories' options
+	NormalizeWindowsPathSeparators(config_map, "allowed_paths");
+	NormalizeWindowsPathSeparators(config_map, "allowed_directories");
+#endif // _WIN32
+
+	// Validate and set all options
 	config.SetOptionsByName(config_map);
 
+	if (!enable_external_access.empty()) {
+		config.SetOptionByName("enable_external_access", duckdb::Value(enable_external_access));
+	}
+
 	bool cache_instance = database != IN_MEMORY_PATH;
 
 	try {
@@ -168,4 +203,8 @@ Connect::Connect(OdbcHandleDbc *dbc_p, string input_str_p) : dbc(dbc_p), input_s
 	}
 	seen_config_options["dsn"] = false;
 	seen_config_options["database"] = false;
+
+	// Required for settings like 'allowed_directories' that use
+	// file separator when checking the property value.
+	config.file_system = duckdb::make_uniq<duckdb::VirtualFileSystem>();
 }

src/connect/connect.cpp

@@ -100,7 +100,7 @@ struct FormatFun {
 struct FormatBytesFun {
 	static constexpr const char *Name = "format_bytes";
 	static constexpr const char *Parameters = "bytes";
-	static constexpr const char *Description = "Converts bytes to a human-readable presentation (e.g. 16000 -> 15.6 KiB)";
+	static constexpr const char *Description = "Converts bytes to a human-readable presentation (e.g., 16000 -> 15.6 KiB)";
 	static constexpr const char *Example = "format_bytes(1000 * 16)";
 	static constexpr const char *Categories = "";
 
@@ -116,7 +116,7 @@ struct FormatreadablesizeFun {
 struct FormatreadabledecimalsizeFun {
 	static constexpr const char *Name = "formatReadableDecimalSize";
 	static constexpr const char *Parameters = "bytes";
-	static constexpr const char *Description = "Converts bytes to a human-readable presentation (e.g. 16000 -> 16.0 KB)";
+	static constexpr const char *Description = "Converts bytes to a human-readable presentation (e.g., 16000 -> 16.0 KB)";
 	static constexpr const char *Example = "format_bytes(1000 * 16)";
 	static constexpr const char *Categories = "";
 

src/duckdb/extension/core_functions/include/core_functions/scalar/string_functions.hpp

@@ -57,7 +57,7 @@ static void MapExtractValueFunc(DataChunk &args, ExpressionState &state, Vector
 
 	// Collect the matching positions
 	Vector pos_vec(LogicalType::INTEGER, count);
-	ListSearchOp<true>(map_vec, key_vec, arg_vec, pos_vec, args.size());
+	ListSearchOp<int32_t>(map_vec, key_vec, arg_vec, pos_vec, args.size());
 
 	UnifiedVectorFormat pos_format;
 	UnifiedVectorFormat lst_format;
@@ -68,7 +68,6 @@ static void MapExtractValueFunc(DataChunk &args, ExpressionState &state, Vector
 	const auto pos_data = UnifiedVectorFormat::GetData<int32_t>(pos_format);
 	const auto inc_list_data = ListVector::GetData(map_vec);
 
-	auto &result_validity = FlatVector::Validity(result);
 	for (idx_t row_idx = 0; row_idx < count; row_idx++) {
 		auto lst_idx = lst_format.sel->get_index(row_idx);
 		if (!lst_format.validity.RowIsValid(lst_idx)) {
@@ -79,7 +78,7 @@ static void MapExtractValueFunc(DataChunk &args, ExpressionState &state, Vector
 		const auto pos_idx = pos_format.sel->get_index(row_idx);
 		if (!pos_format.validity.RowIsValid(pos_idx)) {
 			// We didnt find the key in the map, so return NULL
-			result_validity.SetInvalid(row_idx);
+			FlatVector::SetNull(result, row_idx, true);
 			continue;
 		}
 
@@ -118,7 +117,7 @@ static void MapExtractListFunc(DataChunk &args, ExpressionState &state, Vector &
 
 	// Collect the matching positions
 	Vector pos_vec(LogicalType::INTEGER, count);
-	ListSearchOp<true>(map_vec, key_vec, arg_vec, pos_vec, args.size());
+	ListSearchOp<int32_t>(map_vec, key_vec, arg_vec, pos_vec, args.size());
 
 	UnifiedVectorFormat val_format;
 	UnifiedVectorFormat pos_format;

src/duckdb/extension/core_functions/scalar/map/map_extract.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/common/operator/decimal_cast_operators.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/common/algorithm.hpp"
 #include "duckdb/common/likely.hpp"
@@ -566,6 +567,34 @@ struct RoundDecimalOperator {
 	}
 };
 
+struct RoundIntegerOperator {
+	template <class TA, class TB, class TR>
+	static inline TR Operation(TA input, TB precision) {
+		if (precision < 0) {
+			//	Do all the arithmetic at higher precision
+			using POWERS_OF_TEN_CLASS = typename DecimalCastTraits<TA>::POWERS_OF_TEN_CLASS;
+			if (precision <= -POWERS_OF_TEN_CLASS::CACHED_POWERS_OF_TEN) {
+				return 0;
+			}
+			const auto power_of_ten = POWERS_OF_TEN_CLASS::POWERS_OF_TEN[-precision];
+			auto addition = power_of_ten / 2;
+			if (input < 0) {
+				addition = -addition;
+			}
+			addition += input;
+			addition /= power_of_ten;
+			if (addition) {
+				return UnsafeNumericCast<TR>(addition * power_of_ten);
+			} else {
+				return 0;
+			}
+		} else {
+			//	Rounding integers to higher precision is a NOP
+			return input;
+		}
+	}
+};
+
 struct RoundPrecisionFunctionData : public FunctionData {
 	explicit RoundPrecisionFunctionData(int32_t target_scale) : target_scale(target_scale) {
 	}
@@ -698,10 +727,6 @@ ScalarFunctionSet RoundFun::GetFunctions() {
 		scalar_function_t round_func = nullptr;
 		bind_scalar_function_t bind_func = nullptr;
 		bind_scalar_function_t bind_prec_func = nullptr;
-		if (type.IsIntegral()) {
-			// no round for integral numbers
-			continue;
-		}
 		switch (type.id()) {
 		case LogicalTypeId::FLOAT:
 			round_func = ScalarFunction::UnaryFunction<float, float, RoundOperator>;
@@ -715,7 +740,31 @@ ScalarFunctionSet RoundFun::GetFunctions() {
 			bind_func = BindGenericRoundFunctionDecimal<RoundDecimalOperator>;
 			bind_prec_func = BindDecimalRoundPrecision;
 			break;
+		case LogicalTypeId::TINYINT:
+			round_func = ScalarFunction::NopFunction;
+			round_prec_func = ScalarFunction::BinaryFunction<int8_t, int32_t, int8_t, RoundIntegerOperator>;
+			break;
+		case LogicalTypeId::SMALLINT:
+			round_func = ScalarFunction::NopFunction;
+			round_prec_func = ScalarFunction::BinaryFunction<int16_t, int32_t, int16_t, RoundIntegerOperator>;
+			break;
+		case LogicalTypeId::INTEGER:
+			round_func = ScalarFunction::NopFunction;
+			round_prec_func = ScalarFunction::BinaryFunction<int32_t, int32_t, int32_t, RoundIntegerOperator>;
+			break;
+		case LogicalTypeId::BIGINT:
+			round_func = ScalarFunction::NopFunction;
+			round_prec_func = ScalarFunction::BinaryFunction<int64_t, int32_t, int64_t, RoundIntegerOperator>;
+			break;
+		case LogicalTypeId::HUGEINT:
+			round_func = ScalarFunction::NopFunction;
+			round_prec_func = ScalarFunction::BinaryFunction<hugeint_t, int32_t, hugeint_t, RoundIntegerOperator>;
+			break;
 		default:
+			if (type.IsIntegral()) {
+				// no round for integral numbers
+				continue;
+			}
 			throw InternalException("Unimplemented numeric type for function \"floor\"");
 		}
 		round.AddFunction(ScalarFunction({type}, type, round_func, bind_func));

src/duckdb/extension/core_functions/scalar/math/numeric.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/main/extension_util.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/function/table_function.hpp"
-#include "duckdb/parser/parsed_data/create_table_function_info.hpp"
 #include "include/icu-datefunc.hpp"
 #include "unicode/calendar.h"
 #include "tz_calendar.hpp"
@@ -17,10 +17,10 @@ struct ICUTableRange {
 	struct ICURangeBindData : public TableFunctionData {
 		ICURangeBindData(const ICURangeBindData &other)
 		    : TableFunctionData(other), tz_setting(other.tz_setting), cal_setting(other.cal_setting),
-		      calendar(other.calendar->clone()) {
+		      calendar(other.calendar->clone()), cardinality(other.cardinality) {
 		}
 
-		explicit ICURangeBindData(ClientContext &context) {
+		explicit ICURangeBindData(ClientContext &context, const vector<Value> &inputs) {
 			Value tz_value;
 			if (context.TryGetCurrentSetting("TimeZone", tz_value)) {
 				tz_setting = tz_value.ToString();
@@ -43,11 +43,36 @@ struct ICUTableRange {
 			if (U_FAILURE(success)) {
 				throw InternalException("Unable to create ICU calendar.");
 			}
+
+			timestamp_tz_t bounds[2];
+			interval_t step;
+			for (idx_t i = 0; i < inputs.size(); i++) {
+				if (inputs[i].IsNull()) {
+					return;
+				}
+				if (i >= 2) {
+					step = inputs[i].GetValue<interval_t>();
+				} else {
+					bounds[i] = inputs[i].GetValue<timestamp_tz_t>();
+				}
+			}
+			// Estimate cardinality using micros.
+			int64_t increment = 0;
+			if (!Interval::TryGetMicro(step, increment) || !increment) {
+				return;
+			}
+			int64_t delta = 0;
+			if (!TrySubtractOperator::Operation(bounds[1].value, bounds[0].value, delta)) {
+				return;
+			}
+
+			cardinality = idx_t(delta / increment);
 		}
 
 		string tz_setting;
 		string cal_setting;
 		CalendarPtr calendar;
+		idx_t cardinality;
 	};
 
 	struct ICURangeLocalState : public LocalTableFunctionState {
@@ -130,7 +155,7 @@ struct ICUTableRange {
 	template <bool GENERATE_SERIES>
 	static unique_ptr<FunctionData> Bind(ClientContext &context, TableFunctionBindInput &input,
 	                                     vector<LogicalType> &return_types, vector<string> &names) {
-		auto result = make_uniq<ICURangeBindData>(context);
+		auto result = make_uniq<ICURangeBindData>(context, input.inputs);
 
 		return_types.push_back(LogicalType::TIMESTAMP_TZ);
 		if (GENERATE_SERIES) {
@@ -147,6 +172,14 @@ struct ICUTableRange {
 		return make_uniq<ICURangeLocalState>();
 	}
 
+	static unique_ptr<NodeStatistics> Cardinality(ClientContext &context, const FunctionData *bind_data_p) {
+		if (!bind_data_p) {
+			return nullptr;
+		}
+		auto &bind_data = bind_data_p->Cast<ICURangeBindData>();
+		return make_uniq<NodeStatistics>(bind_data.cardinality, bind_data.cardinality);
+	}
+
 	template <bool GENERATE_SERIES>
 	static OperatorResultType ICUTableRangeFunction(ExecutionContext &context, TableFunctionInput &data_p,
 	                                                DataChunk &input, DataChunk &output) {
@@ -201,6 +234,7 @@ struct ICUTableRange {
 		TableFunction range_function({LogicalType::TIMESTAMP_TZ, LogicalType::TIMESTAMP_TZ, LogicalType::INTERVAL},
 		                             nullptr, Bind<false>, nullptr, RangeDateTimeLocalInit);
 		range_function.in_out_function = ICUTableRangeFunction<false>;
+		range_function.cardinality = Cardinality;
 		range.AddFunction(range_function);
 		ExtensionUtil::RegisterFunction(db, range);
 
@@ -210,6 +244,7 @@ struct ICUTableRange {
 		    {LogicalType::TIMESTAMP_TZ, LogicalType::TIMESTAMP_TZ, LogicalType::INTERVAL}, nullptr, Bind<true>, nullptr,
 		    RangeDateTimeLocalInit);
 		generate_series_function.in_out_function = ICUTableRangeFunction<true>;
+		generate_series_function.cardinality = Cardinality;
 		generate_series.AddFunction(generate_series_function);
 		ExtensionUtil::RegisterFunction(db, generate_series);
 	}