Skip to content

Commit f32828a

Browse files
krlmlrkrlmlr
committed
vendor: Update vendored sources to duckdb/duckdb@9aa45eb
Implement reading concatenated GZIP members (duckdb/duckdb#17255) Update Friendlier SQL link.md (duckdb/duckdb#17248) Only notify JDBC when all runs are successful (duckdb/duckdb#17233)
1 parent 50843f3 commit f32828a

File tree

2 files changed

+68
-40
lines changed

2 files changed

+68
-40
lines changed

‎src/duckdb/src/function/table/version/pragma_version.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#ifndef DUCKDB_PATCH_VERSION
2-
#define DUCKDB_PATCH_VERSION "0-dev2784"
2+
#define DUCKDB_PATCH_VERSION "0-dev2790"
33
#endif
44
#ifndef DUCKDB_MINOR_VERSION
55
#define DUCKDB_MINOR_VERSION 3
@@ -8,10 +8,10 @@
88
#define DUCKDB_MAJOR_VERSION 1
99
#endif
1010
#ifndef DUCKDB_VERSION
11-
#define DUCKDB_VERSION "v1.3.0-dev2784"
11+
#define DUCKDB_VERSION "v1.3.0-dev2790"
1212
#endif
1313
#ifndef DUCKDB_SOURCE_ID
14-
#define DUCKDB_SOURCE_ID "53b5bfc466"
14+
#define DUCKDB_SOURCE_ID "9aa45ebd94"
1515
#endif
1616
#include "duckdb/function/table/system_functions.hpp"
1717
#include "duckdb/main/database.hpp"

‎src/duckdb/third_party/miniz/miniz_wrapper.hpp‎

Lines changed: 65 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,7 @@
1414

1515
namespace duckdb {
1616

17-
enum class MiniZStreamType {
18-
MINIZ_TYPE_NONE,
19-
MINIZ_TYPE_INFLATE,
20-
MINIZ_TYPE_DEFLATE
21-
};
17+
enum class MiniZStreamType { MINIZ_TYPE_NONE, MINIZ_TYPE_INFLATE, MINIZ_TYPE_DEFLATE };
2218

2319
struct MiniZStream {
2420
static constexpr uint8_t GZIP_HEADER_MINSIZE = 10;
@@ -28,10 +24,11 @@ struct MiniZStream {
2824

2925
public:
3026
MiniZStream() : type(MiniZStreamType::MINIZ_TYPE_NONE) {
31-
memset(&stream, 0, sizeof(duckdb_miniz::mz_stream));
27+
ResetStreamInternal();
3228
}
29+
3330
~MiniZStream() {
34-
switch(type) {
31+
switch (type) {
3532
case MiniZStreamType::MINIZ_TYPE_INFLATE:
3633
duckdb_miniz::mz_inflateEnd(&stream);
3734
break;
@@ -42,42 +39,66 @@ struct MiniZStream {
4239
break;
4340
}
4441
}
45-
void FormatException(std::string error_msg) {
42+
43+
void FormatException(const std::string &error_msg) {
4644
throw std::runtime_error(error_msg);
4745
}
46+
4847
void FormatException(const char *error_msg, int mz_ret) {
4948
auto err = duckdb_miniz::mz_error(mz_ret);
5049
FormatException(error_msg + std::string(": ") + (err ? err : "Unknown error code"));
5150
}
51+
5252
void Decompress(const char *compressed_data, size_t compressed_size, char *out_data, size_t out_size) {
53-
auto mz_ret = mz_inflateInit2(&stream, -MZ_DEFAULT_WINDOW_BITS);
54-
if (mz_ret != duckdb_miniz::MZ_OK) {
55-
FormatException("Failed to initialize miniz", mz_ret);
56-
}
5753
type = MiniZStreamType::MINIZ_TYPE_INFLATE;
5854

59-
if (compressed_size < GZIP_HEADER_MINSIZE) {
60-
FormatException("Failed to decompress GZIP block: compressed size is less than gzip header size");
61-
}
62-
auto gzip_hdr = (const unsigned char *)compressed_data;
63-
if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B || gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE ||
64-
gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
65-
FormatException("Input is invalid/unsupported GZIP stream");
66-
}
67-
68-
stream.next_in = (const unsigned char *)compressed_data + GZIP_HEADER_MINSIZE;
69-
stream.avail_in = static_cast<unsigned int >(compressed_size - GZIP_HEADER_MINSIZE);
70-
stream.next_out = (unsigned char *)out_data;
71-
stream.avail_out = static_cast<unsigned int >(out_size);
72-
73-
mz_ret = mz_inflate(&stream, duckdb_miniz::MZ_FINISH);
74-
if (mz_ret != duckdb_miniz::MZ_OK && mz_ret != duckdb_miniz::MZ_STREAM_END) {
75-
FormatException("Failed to decompress GZIP block", mz_ret);
55+
// Loop over blocks
56+
while (compressed_size > 0) {
57+
// Read block header
58+
if (compressed_size < GZIP_HEADER_MINSIZE) {
59+
FormatException("Failed to decompress GZIP block: compressed size is less than gzip header size");
60+
}
61+
auto gzip_hdr = reinterpret_cast<const unsigned char *>(compressed_data);
62+
if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B || gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE ||
63+
gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
64+
FormatException("Input is invalid/unsupported GZIP stream");
65+
}
66+
compressed_data += GZIP_HEADER_MINSIZE;
67+
compressed_size -= GZIP_HEADER_MINSIZE;
68+
69+
// Initialize stream
70+
auto mz_ret = mz_inflateInit2(&stream, -MZ_DEFAULT_WINDOW_BITS);
71+
if (mz_ret != duckdb_miniz::MZ_OK) {
72+
FormatException("Failed to initialize miniz", mz_ret);
73+
}
74+
75+
// Set up in/out
76+
stream.next_in = reinterpret_cast<const unsigned char *>(compressed_data);
77+
stream.avail_in = static_cast<unsigned int>(compressed_size);
78+
stream.next_out = reinterpret_cast<unsigned char *>(out_data);
79+
stream.avail_out = static_cast<unsigned int>(out_size);
80+
81+
// Decompress and uninitialize stream
82+
mz_ret = mz_inflate(&stream, duckdb_miniz::MZ_FINISH);
83+
if (mz_ret != duckdb_miniz::MZ_OK && mz_ret != duckdb_miniz::MZ_STREAM_END) {
84+
FormatException("Failed to decompress GZIP block", mz_ret);
85+
}
86+
mz_inflateEnd(&stream);
87+
88+
// Update indices
89+
compressed_data += GZIP_FOOTER_SIZE + stream.total_in;
90+
compressed_size -= GZIP_FOOTER_SIZE + stream.total_in;
91+
out_data += stream.total_out;
92+
out_size -= stream.total_out;
93+
94+
ResetStreamInternal();
7695
}
7796
}
78-
size_t MaxCompressedLength(size_t input_size) {
97+
98+
static size_t MaxCompressedLength(size_t input_size) {
7999
return duckdb_miniz::mz_compressBound(input_size) + GZIP_HEADER_MINSIZE + GZIP_FOOTER_SIZE;
80100
}
101+
81102
static void InitializeGZIPHeader(unsigned char *gzip_header) {
82103
memset(gzip_header, 0, GZIP_HEADER_MINSIZE);
83104
gzip_header[0] = 0x1F;
@@ -104,36 +125,43 @@ struct MiniZStream {
104125
}
105126

106127
void Compress(const char *uncompressed_data, size_t uncompressed_size, char *out_data, size_t *out_size) {
107-
auto mz_ret = mz_deflateInit2(&stream, duckdb_miniz::MZ_DEFAULT_LEVEL, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 1, 0);
128+
auto mz_ret =
129+
mz_deflateInit2(&stream, duckdb_miniz::MZ_DEFAULT_LEVEL, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 1, 0);
108130
if (mz_ret != duckdb_miniz::MZ_OK) {
109131
FormatException("Failed to initialize miniz", mz_ret);
110132
}
111133
type = MiniZStreamType::MINIZ_TYPE_DEFLATE;
112134

113-
auto gzip_header = (unsigned char*) out_data;
135+
auto gzip_header = reinterpret_cast<unsigned char *>(out_data);
114136
InitializeGZIPHeader(gzip_header);
115137

116138
auto gzip_body = gzip_header + GZIP_HEADER_MINSIZE;
117139

118-
stream.next_in = (const unsigned char*) uncompressed_data;
119-
stream.avail_in = static_cast<unsigned int >(uncompressed_size);
140+
stream.next_in = reinterpret_cast<const unsigned char *>(uncompressed_data);
141+
stream.avail_in = static_cast<unsigned int>(uncompressed_size);
120142
stream.next_out = gzip_body;
121-
stream.avail_out = static_cast<unsigned int >(*out_size - GZIP_HEADER_MINSIZE);
143+
stream.avail_out = static_cast<unsigned int>(*out_size - GZIP_HEADER_MINSIZE);
122144

123145
mz_ret = mz_deflate(&stream, duckdb_miniz::MZ_FINISH);
124146
if (mz_ret != duckdb_miniz::MZ_OK && mz_ret != duckdb_miniz::MZ_STREAM_END) {
125147
FormatException("Failed to compress GZIP block", mz_ret);
126148
}
127149
auto gzip_footer = gzip_body + stream.total_out;
128-
auto crc = duckdb_miniz::mz_crc32(MZ_CRC32_INIT, (const unsigned char*) uncompressed_data, uncompressed_size);
150+
auto crc = duckdb_miniz::mz_crc32(MZ_CRC32_INIT, reinterpret_cast<const unsigned char *>(uncompressed_data),
151+
uncompressed_size);
129152
InitializeGZIPFooter(gzip_footer, crc, uncompressed_size);
130153

131154
*out_size = stream.total_out + GZIP_HEADER_MINSIZE + GZIP_FOOTER_SIZE;
132155
}
133156

157+
private:
158+
void ResetStreamInternal() {
159+
memset(&stream, 0, sizeof(duckdb_miniz::mz_stream));
160+
}
161+
134162
private:
135163
duckdb_miniz::mz_stream stream;
136164
MiniZStreamType type;
137165
};
138166

139-
}
167+
} // namespace duckdb

0 commit comments

Comments
 (0)