Skip to content

Commit 2c6fa8f

Browse files
krlmlrkrlmlr
committed
vendor: Update vendored sources to duckdb/duckdb@ad27205
Merge v1.3-ossivalis into main (duckdb/duckdb#17690)
1 parent 7942b91 commit 2c6fa8f

File tree

69 files changed

+652
-258
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+652
-258
lines changed

‎patch/0018-uninitialized.patch‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎patch/0019-uninitialized.patch‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎src/duckdb/extension/core_functions/scalar/random/random.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ struct ExtractTimestampUuidOperator {
2929
}
3030

3131
// UUID v7 begins with a 48 bit big-endian Unix Epoch timestamp with millisecond granularity.
32-
const int64_t upper = input.upper;
32+
int64_t upper = input.upper;
33+
// flip the top byte
34+
upper ^= NumericLimits<int64_t>::Minimum();
3335
int64_t unix_ts_milli = upper;
3436
unix_ts_milli = unix_ts_milli >> 16;
3537

‎src/duckdb/extension/parquet/include/thrift_tools.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ struct ReadAheadBuffer {
119119
throw std::runtime_error("Prefetch registered requested for bytes outside file");
120120
}
121121
read_head.buffer_handle = file_handle.Read(read_head.buffer_ptr, read_head.size, read_head.location);
122+
D_ASSERT(read_head.buffer_handle.IsValid());
122123
read_head.data_isset = true;
123124
}
124125
}
@@ -141,8 +142,10 @@ class ThriftFileTransport : public duckdb_apache::thrift::transport::TVirtualTra
141142
if (!prefetch_buffer->data_isset) {
142143
prefetch_buffer->buffer_handle =
143144
file_handle.Read(prefetch_buffer->buffer_ptr, prefetch_buffer->size, prefetch_buffer->location);
145+
D_ASSERT(prefetch_buffer->buffer_handle.IsValid());
144146
prefetch_buffer->data_isset = true;
145147
}
148+
D_ASSERT(prefetch_buffer->buffer_handle.IsValid());
146149
memcpy(buf, prefetch_buffer->buffer_ptr + location - prefetch_buffer->location, len);
147150
} else if (prefetch_mode && len < PREFETCH_FALLBACK_BUFFERSIZE && len > 0) {
148151
Prefetch(location, MinValue<uint64_t>(PREFETCH_FALLBACK_BUFFERSIZE, file_handle.GetFileSize() - location));

‎src/duckdb/extension/parquet/parquet_reader.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1116,7 +1116,7 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat
11161116
state.prefetch_mode = false;
11171117
}
11181118

1119-
state.file_handle = fs.OpenFile(file_handle->GetPath(), flags);
1119+
state.file_handle = fs.OpenFile(file, flags);
11201120
}
11211121
state.adaptive_filter.reset();
11221122
state.scan_filters.clear();

‎src/duckdb/extension/parquet/reader/struct_column_reader.cpp‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,21 @@ static bool TypeHasExactRowCount(const LogicalType &type) {
118118
}
119119

120120
idx_t StructColumnReader::GroupRowsAvailable() {
121-
for (idx_t i = 0; i < child_readers.size(); i++) {
122-
if (TypeHasExactRowCount(child_readers[i]->Type())) {
123-
return child_readers[i]->GroupRowsAvailable();
121+
for (auto &child : child_readers) {
122+
if (!child) {
123+
continue;
124+
}
125+
if (TypeHasExactRowCount(child->Type())) {
126+
return child->GroupRowsAvailable();
127+
}
128+
}
129+
for (auto &child : child_readers) {
130+
if (!child) {
131+
continue;
124132
}
133+
return child->GroupRowsAvailable();
125134
}
126-
return child_readers[0]->GroupRowsAvailable();
135+
throw InternalException("No projected columns in struct?");
127136
}
128137

129138
} // namespace duckdb

‎src/duckdb/src/common/local_file_system.cpp‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -838,21 +838,21 @@ static string AdditionalLockInfo(const std::wstring path) {
838838

839839
status = RmStartSession(&session, 0, session_key);
840840
if (status != ERROR_SUCCESS) {
841-
return "";
841+
return string();
842842
}
843843

844844
PCWSTR path_ptr = path.c_str();
845845
status = RmRegisterResources(session, 1, &path_ptr, 0, NULL, 0, NULL);
846846
if (status != ERROR_SUCCESS) {
847-
return "";
847+
return string();
848848
}
849849
UINT process_info_size_needed, process_info_size;
850850

851851
// we first call with nProcInfo = 0 to find out how much to allocate
852852
process_info_size = 0;
853853
status = RmGetList(session, &process_info_size_needed, &process_info_size, NULL, &reason);
854854
if (status != ERROR_MORE_DATA || process_info_size_needed == 0) {
855-
return "";
855+
return string();
856856
}
857857

858858
// allocate
@@ -866,8 +866,7 @@ static string AdditionalLockInfo(const std::wstring path) {
866866
return "";
867867
}
868868

869-
string conflict_string = "File is already open in ";
870-
869+
string conflict_string;
871870
for (UINT process_idx = 0; process_idx < process_info_size; process_idx++) {
872871
string process_name = WindowsUtil::UnicodeToUTF8(process_info[process_idx].strAppName);
873872
auto pid = process_info[process_idx].Process.dwProcessId;
@@ -886,7 +885,10 @@ static string AdditionalLockInfo(const std::wstring path) {
886885
}
887886

888887
RmEndSession(session);
889-
return conflict_string;
888+
if (conflict_string.empty()) {
889+
return string();
890+
}
891+
return "File is already open in " + conflict_string;
890892
}
891893

892894
bool LocalFileSystem::IsPrivateFile(const string &path_p, FileOpener *opener) {
@@ -950,12 +952,11 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, FileOpenF
950952
}
951953
auto error = LocalFileSystem::GetLastErrorAsString();
952954

953-
auto better_error = AdditionalLockInfo(unicode_path);
954-
if (!better_error.empty()) {
955-
throw IOException(better_error);
956-
} else {
957-
throw IOException("Cannot open file \"%s\": %s", path.c_str(), error);
955+
auto extended_error = AdditionalLockInfo(unicode_path);
956+
if (!extended_error.empty()) {
957+
extended_error = "\n" + extended_error;
958958
}
959+
throw IOException("Cannot open file \"%s\": %s%s", path.c_str(), error, extended_error);
959960
}
960961
auto handle = make_uniq<WindowsFileHandle>(*this, path.c_str(), hFile, flags);
961962
if (flags.OpenForAppending()) {

‎src/duckdb/src/common/multi_file/multi_file_column_mapper.cpp‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ MultiFileColumnMapper::MultiFileColumnMapper(ClientContext &context, MultiFileRe
1717
MultiFileReaderData &reader_data,
1818
const vector<MultiFileColumnDefinition> &global_columns,
1919
const vector<ColumnIndex> &global_column_ids,
20-
optional_ptr<TableFilterSet> filters, const OpenFileInfo &initial_file,
20+
optional_ptr<TableFilterSet> filters, MultiFileList &multi_file_list,
2121
const MultiFileReaderBindData &bind_data,
2222
const virtual_column_map_t &virtual_columns)
23-
: context(context), multi_file_reader(multi_file_reader), reader_data(reader_data), global_columns(global_columns),
24-
global_column_ids(global_column_ids), global_filters(filters), initial_file(initial_file), bind_data(bind_data),
25-
virtual_columns(virtual_columns) {
23+
: context(context), multi_file_reader(multi_file_reader), multi_file_list(multi_file_list),
24+
reader_data(reader_data), global_columns(global_columns), global_column_ids(global_column_ids),
25+
global_filters(filters), bind_data(bind_data), virtual_columns(virtual_columns) {
2626
}
2727

2828
struct MultiFileIndexMapping {
@@ -189,7 +189,8 @@ void MultiFileColumnMapper::ThrowColumnNotFoundError(const string &global_column
189189
"the original file \"%s\", but could not be found in file \"%s\".\nCandidate names: "
190190
"%s\nIf you are trying to "
191191
"read files with different schemas, try setting union_by_name=True",
192-
file_name, global_column_name, initial_file.path, file_name, candidate_names);
192+
file_name, global_column_name, multi_file_list.GetFirstFile().path, file_name,
193+
candidate_names);
193194
}
194195

195196
//! Check if a column is trivially mappable (i.e. the column is effectively identical to the global column)
@@ -296,6 +297,19 @@ ColumnMapResult MapColumnList(ClientContext &context, const MultiFileColumnDefin
296297
result.column_map = Value::STRUCT(std::move(child_list));
297298
}
298299
}
300+
if (is_selected && child_map.default_value) {
301+
// we have default values at a previous level wrap it in a "list"
302+
child_list_t<LogicalType> default_type_list;
303+
default_type_list.emplace_back("list", child_map.default_value->return_type);
304+
vector<unique_ptr<Expression>> default_expressions;
305+
child_map.default_value->alias = "list";
306+
default_expressions.push_back(std::move(child_map.default_value));
307+
auto default_type = LogicalType::STRUCT(std::move(default_type_list));
308+
auto struct_pack_fun = StructPackFun::GetFunction();
309+
auto bind_data = make_uniq<VariableReturnBindData>(default_type);
310+
result.default_value = make_uniq<BoundFunctionExpression>(std::move(default_type), std::move(struct_pack_fun),
311+
std::move(default_expressions), std::move(bind_data));
312+
}
299313
result.column_index = make_uniq<ColumnIndex>(local_id.GetId(), std::move(child_indexes));
300314
result.mapping = std::move(mapping);
301315
return result;
@@ -346,7 +360,7 @@ ColumnMapResult MapColumnMap(ClientContext &context, const MultiFileColumnDefini
346360

347361
auto nested_mapper = mapper.Create(local_key_value.children);
348362
child_list_t<Value> column_mapping;
349-
unique_ptr<Expression> default_expression;
363+
vector<unique_ptr<Expression>> default_expressions;
350364
unordered_map<idx_t, const_reference<ColumnIndex>> selected_children;
351365
if (global_index.HasChildren()) {
352366
//! FIXME: is this expected for maps??
@@ -377,6 +391,10 @@ ColumnMapResult MapColumnMap(ClientContext &context, const MultiFileColumnDefini
377391
// found a column mapping for the component - emplace it
378392
column_mapping.emplace_back(name, std::move(map_result.column_map));
379393
}
394+
if (map_result.default_value) {
395+
map_result.default_value->alias = name;
396+
default_expressions.push_back(std::move(map_result.default_value));
397+
}
380398
}
381399

382400
ColumnMapResult result;
@@ -392,6 +410,18 @@ ColumnMapResult MapColumnMap(ClientContext &context, const MultiFileColumnDefini
392410
result.column_map = Value::STRUCT(std::move(child_list));
393411
}
394412
}
413+
if (!default_expressions.empty()) {
414+
// we have default values at a previous level wrap it in a "list"
415+
child_list_t<LogicalType> default_type_list;
416+
for (auto &expr : default_expressions) {
417+
default_type_list.emplace_back(expr->GetAlias(), expr->return_type);
418+
}
419+
auto default_type = LogicalType::STRUCT(std::move(default_type_list));
420+
auto struct_pack_fun = StructPackFun::GetFunction();
421+
auto bind_data = make_uniq<VariableReturnBindData>(default_type);
422+
result.default_value = make_uniq<BoundFunctionExpression>(std::move(default_type), std::move(struct_pack_fun),
423+
std::move(default_expressions), std::move(bind_data));
424+
}
395425
vector<ColumnIndex> map_indexes;
396426
map_indexes.emplace_back(0, std::move(child_indexes));
397427

@@ -562,11 +592,6 @@ unique_ptr<Expression> ConstructMapExpression(ClientContext &context, idx_t loca
562592
}
563593
return expr;
564594
}
565-
// struct column - generate a remap_struct - but only if we have any columns to remap
566-
if (mapping.column_map.IsNull()) {
567-
// no columns to map - emit the default value directly
568-
return std::move(mapping.default_value);
569-
}
570595
// generate the remap_struct function call
571596
vector<unique_ptr<Expression>> children;
572597
children.push_back(std::move(expr));

‎src/duckdb/src/common/multi_file/multi_file_list.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,10 @@ bool MultiFileList::IsEmpty() {
193193
return GetExpandResult() == FileExpandResult::NO_FILES;
194194
}
195195

196+
unique_ptr<MultiFileList> MultiFileList::Copy() {
197+
return make_uniq<SimpleMultiFileList>(GetAllFiles());
198+
}
199+
196200
//===--------------------------------------------------------------------===//
197201
// SimpleMultiFileList
198202
//===--------------------------------------------------------------------===//

‎src/duckdb/src/common/multi_file/multi_file_reader.cpp‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ unique_ptr<FunctionData> MultiFileBindData::Copy() const {
5656
if (bind_data) {
5757
result->bind_data = unique_ptr_cast<FunctionData, TableFunctionData>(bind_data->Copy());
5858
}
59-
result->file_list = make_uniq<SimpleMultiFileList>(file_list->GetAllFiles());
59+
result->file_list = file_list->Copy();
6060
result->multi_file_reader = multi_file_reader->Copy();
6161
result->interface = interface->Copy();
6262
result->columns = columns;
@@ -371,15 +371,12 @@ MultiFileReader::InitializeGlobalState(ClientContext &context, const MultiFileOp
371371
return nullptr;
372372
}
373373

374-
ReaderInitializeType MultiFileReader::CreateMapping(ClientContext &context, MultiFileReaderData &reader_data,
375-
const vector<MultiFileColumnDefinition> &global_columns,
376-
const vector<ColumnIndex> &global_column_ids,
377-
optional_ptr<TableFilterSet> filters,
378-
const OpenFileInfo &initial_file,
379-
const MultiFileReaderBindData &bind_data,
380-
const virtual_column_map_t &virtual_columns) {
374+
ReaderInitializeType MultiFileReader::CreateMapping(
375+
ClientContext &context, MultiFileReaderData &reader_data, const vector<MultiFileColumnDefinition> &global_columns,
376+
const vector<ColumnIndex> &global_column_ids, optional_ptr<TableFilterSet> filters, MultiFileList &multi_file_list,
377+
const MultiFileReaderBindData &bind_data, const virtual_column_map_t &virtual_columns) {
381378
MultiFileColumnMapper column_mapper(context, *this, reader_data, global_columns, global_column_ids, filters,
382-
initial_file, bind_data, virtual_columns);
379+
multi_file_list, bind_data, virtual_columns);
383380
return column_mapper.CreateMapping();
384381
}
385382

@@ -577,12 +574,11 @@ ReaderInitializeType MultiFileReader::InitializeReader(MultiFileReaderData &read
577574
const vector<MultiFileColumnDefinition> &global_columns,
578575
const vector<ColumnIndex> &global_column_ids,
579576
optional_ptr<TableFilterSet> table_filters,
580-
ClientContext &context,
581-
optional_ptr<MultiFileReaderGlobalState> global_state) {
577+
ClientContext &context, MultiFileGlobalState &gstate) {
582578
FinalizeBind(reader_data, bind_data.file_options, bind_data.reader_bind, global_columns, global_column_ids, context,
583-
global_state);
584-
return CreateMapping(context, reader_data, global_columns, global_column_ids, table_filters,
585-
bind_data.file_list->GetFirstFile(), bind_data.reader_bind, bind_data.virtual_columns);
579+
gstate.multi_file_reader_state.get());
580+
return CreateMapping(context, reader_data, global_columns, global_column_ids, table_filters, gstate.file_list,
581+
bind_data.reader_bind, bind_data.virtual_columns);
586582
}
587583

588584
shared_ptr<BaseFileReader> MultiFileReader::CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,

0 commit comments

Comments
 (0)