From 72d7af31b80994b4fa559c5c8202fe2d196073a2 Mon Sep 17 00:00:00 2001 From: David Beck Date: Wed, 29 Jul 2026 19:30:37 +0100 Subject: [PATCH] Fix crashes with malformed compressed data The Gorilla and Dictionary compressors crashed on malformed data with the reverse decompression iterator and the bulk decompressor. This issue was reported by Mehmet Ince @mdisec https://mehmetince.net/, and the test case binaries are his contribution that I integrated into the fuzzer test suite. This change adds validation for these cases and adds reverse iterator coverage to the fuzzer tests, so the randomised test will catch future issues. It already caught issues in other compressors like deltadelta, which I also fixed in this review. --- .unreleased/pr_10360 | 2 + src/adts/bit_array.h | 1 + src/adts/bit_array_impl.h | 17 +- tsl/src/compression/algorithms/deltadelta.c | 114 +++++++++++++- .../compression/algorithms/deltadelta_impl.c | 5 + tsl/src/compression/algorithms/dictionary.c | 73 ++++++++- tsl/src/compression/algorithms/gorilla.c | 145 +++++++++++++++++- tsl/src/compression/algorithms/gorilla_impl.c | 7 + tsl/src/compression/algorithms/simple8b_rle.h | 14 +- .../compression/algorithms/uuid_compress.c | 5 +- tsl/test/expected/compression_algos.out | 27 ++-- .../00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa | Bin 0 -> 66 bytes .../05a79f06cf3f67f726dae68d18a2290f6c9a50c9 | 1 + .../1015e4b50aba50649bb467930cfe971059d6ae9d | Bin 0 -> 1018 bytes .../174db52f92ceb9f868c11cd0040ca4a20801e6fc | Bin 0 -> 58 bytes .../467225894f603fcf9f2160d548ebc478bc7a4875 | Bin 0 -> 66 bytes .../56527dcf9aa46bf3d1dbe57ad731b921c7eeb004 | Bin 0 -> 58 bytes .../8d51515ae0fdb890a4232de9cd9b1461d83577fa | Bin 0 -> 58 bytes .../98561f0a5f98d0c063be2e5395136833c804b575 | Bin 0 -> 98 bytes .../a78ca6006b321a392f21f7360a4ff7399a0d9e11 | Bin 0 -> 162 bytes .../c99004415eb02cf0e1ffdf80350dcb533d94b4a5 | Bin 0 -> 98 bytes ...h-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 | Bin 0 -> 642 bytes ...h-581aa419c8bb6424793f216d0fda5f1470749f0d | Bin 0 -> 66 bytes ...h-b666969713074b288af5d833bda5647fad29b055 | Bin 0 -> 154 bytes ...h-da39a3ee5e6b4b0d3255bfef95601890afd80709 | 0 ...h-de00eb0e8cb99709d1cf37f4b5e36b4dca7e529c | Bin 0 -> 186 bytes .../d26928947e574611b54da6ee673f98e5bed26f3f | Bin 0 -> 794 bytes .../e3c7308df09bb6c90f477547d7fa90dce9c79629 | Bin 0 -> 114 bytes .../007929c46768d6da44b95e8e8f62cf144be2b649 | Bin 0 -> 193 bytes .../01a73a5c2e29161dd645c094052ebebef2ed2512 | Bin 0 -> 49 bytes .../0371e59314a5f7e3e6525df351df7b0b2353a0d2 | Bin 0 -> 193 bytes .../096864a5697e5508063ed40b3b53f8a631910888 | Bin 0 -> 193 bytes .../098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 | Bin 0 -> 73 bytes .../426effe2b9af1923459a27c89e3c4831417936a6 | Bin 0 -> 50 bytes .../6b3b505c196626a582333b7762c696c031d0c6ab | Bin 0 -> 58 bytes .../855d7a2f02334aa8ef42aee9599ce204c9725417 | Bin 0 -> 145 bytes .../9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 | Bin 0 -> 97 bytes ...h-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc | Bin 0 -> 1854 bytes ...h-6fe984d76d7457879cee2fa63045473c55409ea0 | Bin 0 -> 49 bytes ...h-70c68b0f5b1afee677c690f49a7fcfe0973614b3 | Bin 0 -> 3970 bytes .../crash-pr10360-idx-values-oob | Bin 0 -> 170 bytes .../crash-pr10360-negative-idx | Bin 0 -> 53 bytes .../014436ca4af1061ff896fe78c472312dd723d413 | Bin 0 -> 124 bytes .../02269e63b0856b2e0d7c3d222b61896636377bf4 | Bin 0 -> 124 bytes .../07726c86c6954a77f1ddf28907b67229fe0c6310 | Bin 0 -> 100 bytes .../105503ff3f2307d622ce91aa7d684348abb6170e | Bin 0 -> 100 bytes .../121a9af889bd4ca2266be5a4f680d3bead8d02d6 | 1 + .../1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 | Bin 0 -> 100 bytes .../175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 | Bin 0 -> 124 bytes .../1f070ff69fa02284c29dd02649ae54ef0f597570 | Bin 0 -> 100 bytes .../2187f25c223185c4dc11ed35fcd308d7008eeec2 | Bin 0 -> 100 bytes .../236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb | Bin 0 -> 100 bytes .../2a5f5f7a9bc26e210d02659f238a1781d0974ee5 | Bin 0 -> 116 bytes .../372bbd248990d2654a96083580591ed09ea515a2 | Bin 0 -> 116 bytes .../54dee5a23c3c0b34009bf11057ae923c26f50224 | Bin 0 -> 116 bytes .../8476f031bf50af967fc23b957dcb24802f8e228a | Bin 0 -> 124 bytes .../a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 | 1 + .../b470e82376caa703fcb832a140d3738d3734223c | Bin 0 -> 116 bytes .../c77acb44cc2f309d06b35c132bf96d0a44aedbb3 | Bin 0 -> 100 bytes ...h-0e4e2dc4fef8a8d9272ad0e2530093a2fc527196 | Bin 0 -> 132 bytes ...h-51ea8f10ebab4dfe07d48289eaec8670472dde52 | Bin 0 -> 75 bytes ...h-652069006785dfb361c63ac6db2b8cbb4994510f | Bin 0 -> 244 bytes ...h-687a1db698c5d9b85fa19eaed1572d903d958018 | Bin 0 -> 151 bytes ...h-8edac9852766106ba0a087eeb07c981fc71f1524 | Bin 0 -> 468 bytes ...h-9c7d5d32a7af25fd5d31a83975fcc4d734f17437 | Bin 0 -> 469 bytes ...h-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b | Bin 0 -> 1084 bytes .../crash-pr10360-leading-zeros-oob | Bin 0 -> 92 bytes .../d5f0e700f5e3bc8393cb11dde18c07d38e910bef | Bin 0 -> 260 bytes .../src/decompress_arithmetic_test_impl.c | 58 ++++++- tsl/test/src/decompress_text_test_impl.c | 69 +++++++++ 70 files changed, 504 insertions(+), 36 deletions(-) create mode 100644 .unreleased/pr_10360 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/05a79f06cf3f67f726dae68d18a2290f6c9a50c9 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/1015e4b50aba50649bb467930cfe971059d6ae9d create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/174db52f92ceb9f868c11cd0040ca4a20801e6fc create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/467225894f603fcf9f2160d548ebc478bc7a4875 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/56527dcf9aa46bf3d1dbe57ad731b921c7eeb004 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/crash-581aa419c8bb6424793f216d0fda5f1470749f0d create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/crash-b666969713074b288af5d833bda5647fad29b055 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/crash-da39a3ee5e6b4b0d3255bfef95601890afd80709 create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/crash-de00eb0e8cb99709d1cf37f4b5e36b4dca7e529c create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/d26928947e574611b54da6ee673f98e5bed26f3f create mode 100644 tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/0371e59314a5f7e3e6525df351df7b0b2353a0d2 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/096864a5697e5508063ed40b3b53f8a631910888 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab create mode 100644 tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc create mode 100644 tsl/test/fuzzing/compression/dictionary-text/crash-6fe984d76d7457879cee2fa63045473c55409ea0 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/crash-70c68b0f5b1afee677c690f49a7fcfe0973614b3 create mode 100644 tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob create mode 100644 tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/07726c86c6954a77f1ddf28907b67229fe0c6310 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/2187f25c223185c4dc11ed35fcd308d7008eeec2 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/c77acb44cc2f309d06b35c132bf96d0a44aedbb3 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-0e4e2dc4fef8a8d9272ad0e2530093a2fc527196 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-51ea8f10ebab4dfe07d48289eaec8670472dde52 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-652069006785dfb361c63ac6db2b8cbb4994510f create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-9c7d5d32a7af25fd5d31a83975fcc4d734f17437 create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob create mode 100644 tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef diff --git a/.unreleased/pr_10360 b/.unreleased/pr_10360 new file mode 100644 index 00000000000..25b032d17d3 --- /dev/null +++ b/.unreleased/pr_10360 @@ -0,0 +1,2 @@ +Fixes: #10360 Decompressor crashes with malformed compressed data +Thanks: @mdisec for reporting the issues with the validation of the compressed data during decompression diff --git a/src/adts/bit_array.h b/src/adts/bit_array.h index 7125b0c7be5..03a571453b7 100644 --- a/src/adts/bit_array.h +++ b/src/adts/bit_array.h @@ -37,6 +37,7 @@ pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *i static void bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array); /* return last num_bits in forward-order (not reverse-order); must have been written as num_bits */ static uint64 bit_array_iter_next_rev(BitArrayIterator *iter, uint8 num_bits); +static inline uint64 bit_array_iter_position(const BitArrayIterator *iter); /* I/O */ static inline void bit_array_send(StringInfo buffer, const BitArray *data); diff --git a/src/adts/bit_array_impl.h b/src/adts/bit_array_impl.h index 1a166f2aa3c..8de12730b95 100644 --- a/src/adts/bit_array_impl.h +++ b/src/adts/bit_array_impl.h @@ -85,8 +85,10 @@ bit_array_num_buckets(const BitArray *array) static inline uint64 bit_array_num_bits(const BitArray *array) { - return (BITS_PER_BUCKET * (array->buckets.num_elements - UINT64CONST(1))) + - array->bits_used_in_last_bucket; + return array->buckets.num_elements == 0 ? + 0 : + (BITS_PER_BUCKET * (array->buckets.num_elements - UINT64CONST(1))) + + array->bits_used_in_last_bucket; } static inline uint64 * @@ -250,6 +252,15 @@ bit_array_iterator_init(BitArrayIterator *iter, const BitArray *array) }; } +static inline uint64 +bit_array_iter_position(const BitArrayIterator *iter) +{ + Assert(iter->current_bucket >= 0); + return iter->current_bucket < 0 ? + 0 : + (BITS_PER_BUCKET * iter->current_bucket) + iter->bits_used_in_current_bucket; +} + pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *iter, uint8 num_bits) { @@ -302,7 +313,7 @@ bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array) { *iter = (BitArrayIterator){ .array = array, - .current_bucket = array->buckets.num_elements - 1, + .current_bucket = array->buckets.num_elements == 0 ? 0 : array->buckets.num_elements - 1, .bits_used_in_current_bucket = array->bits_used_in_last_bucket, }; } diff --git a/tsl/src/compression/algorithms/deltadelta.c b/tsl/src/compression/algorithms/deltadelta.c index 10e57d1936f..12efead3be5 100644 --- a/tsl/src/compression/algorithms/deltadelta.c +++ b/tsl/src/compression/algorithms/deltadelta.c @@ -60,6 +60,11 @@ typedef struct DeltaDeltaDecompressionIterator Simple8bRleDecompressionIterator delta_deltas; Simple8bRleDecompressionIterator nulls; bool has_nulls; + + /* for validating the last_value set in the header + * and cross check with the iterator */ + uint64 last_value_to_return; + uint64 last_returned_value; } DeltaDeltaDecompressionIterator; typedef struct DeltaDeltaCompressor @@ -481,6 +486,44 @@ delta_delta_compressor_append_value(DeltaDeltaCompressor *compressor, int64 next /**********************************************************************************/ /**********************************************************************************/ +static inline uint32 +decompression_iterator_items_seen(const DeltaDeltaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->delta_deltas.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const DeltaDeltaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->delta_deltas.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const DeltaDeltaDecompressionIterator *iter) +{ + return iter->delta_deltas.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const DeltaDeltaDecompressionIterator *iter) +{ + return iter->delta_deltas.num_elements; +} + static void int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) @@ -507,6 +550,8 @@ int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, .prev_val = 0, .prev_delta = 0, .has_nulls = has_nulls, + .last_returned_value = 0, + .last_value_to_return = header->last_value, }; simple8brle_decompression_iterator_init_forward(&iter->delta_deltas, deltas); @@ -515,9 +560,13 @@ int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_forward(&iter->nulls, nulls); + CheckCompressedData(deltas->num_elements <= nulls->num_elements); } } +static DecompressResultInternal +delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompressionIterator *iter); + static void int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) @@ -529,7 +578,7 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, DeltaDeltaCompressed *header = consumeCompressedData(&si, sizeof(DeltaDeltaCompressed)); Simple8bRleSerialized *deltas = bytes_deserialize_simple8b_and_advance(&si); - Assert(header->has_nulls == 0 || header->has_nulls == 1); + CheckCompressedData(header->has_nulls == 0 || header->has_nulls == 1); *iter = (DeltaDeltaDecompressionIterator){ .base = { @@ -541,6 +590,8 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, .prev_val = header->last_value, .prev_delta = header->last_delta, .has_nulls = header->has_nulls, + .last_returned_value = 0, + .last_value_to_return = 0, }; simple8brle_decompression_iterator_init_reverse(&iter->delta_deltas, deltas); @@ -549,6 +600,31 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_reverse(&iter->nulls, nulls); + CheckCompressedData(deltas->num_elements <= nulls->num_elements); + } + + /* on reverse iteration the `last_value` in the header is critical for + * the rest of the decoding. when we receive malformed data we can only + * cross reference the result with the first value + */ + { + DeltaDeltaDecompressionIterator forward_iter; + forward_iter.base.compression_algorithm = COMPRESSION_ALGORITHM_DELTADELTA; + forward_iter.base.forward = true; + forward_iter.base.element_type = element_type; + forward_iter.base.try_next = delta_delta_decompression_iterator_try_next_forward; + int64_decompression_iterator_init_forward(&forward_iter, compressed, element_type); + /* find the first non-null value */ + DecompressResultInternal first_val = + delta_delta_decompression_iterator_try_next_forward_internal(&forward_iter); + int32 n = 0; + while (first_val.is_done == false && first_val.is_null) + { + CheckCompressedData(n++ < GLOBAL_MAX_ROWS_PER_COMPRESSION); + first_val = delta_delta_decompression_iterator_try_next_forward_internal(&forward_iter); + } + CheckCompressedData(!first_val.is_done); + iter->last_value_to_return = first_val.val; } } @@ -615,6 +691,14 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -633,6 +717,14 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -642,6 +734,7 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres iter->prev_delta += delta_delta; iter->prev_val += iter->prev_delta; + iter->last_returned_value = iter->prev_val; return (DecompressResultInternal){ .val = iter->prev_val, @@ -712,6 +805,14 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -719,7 +820,7 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres if (result.val != 0) { - Assert(result.val == 1); + CheckCompressedData(result.val == 1); return (DecompressResultInternal){ .is_null = true, }; @@ -730,6 +831,14 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -740,6 +849,7 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres delta_delta = zig_zag_decode(result.val); iter->prev_val -= iter->prev_delta; iter->prev_delta -= delta_delta; + iter->last_returned_value = val; return (DecompressResultInternal){ .val = val, diff --git a/tsl/src/compression/algorithms/deltadelta_impl.c b/tsl/src/compression/algorithms/deltadelta_impl.c index 78fd29fa06d..99c0f242ab3 100644 --- a/tsl/src/compression/algorithms/deltadelta_impl.c +++ b/tsl/src/compression/algorithms/deltadelta_impl.c @@ -90,6 +90,7 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory #undef INNER_LOOP_SIZE uint64 *restrict validity_bitmap = NULL; + ELEMENT_TYPE last_value = decompressed_values[n_total - 1]; if (has_nulls) { /* Now move the data to account for nulls, and fill the validity bitmap. */ @@ -117,6 +118,7 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; + last_value = decompressed_values[current_notnull_element]; for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); @@ -136,6 +138,9 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory Assert(current_notnull_element == -1); } + /* the last value in the header must match with the last that we returned */ + CheckCompressedData(last_value == (ELEMENT_TYPE) header->last_value); + /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; diff --git a/tsl/src/compression/algorithms/dictionary.c b/tsl/src/compression/algorithms/dictionary.c index 571ec23aace..29228a1c65d 100644 --- a/tsl/src/compression/algorithms/dictionary.c +++ b/tsl/src/compression/algorithms/dictionary.c @@ -416,6 +416,44 @@ dictionary_compressor_finish(DictionaryCompressor *compressor) /// Decompressor /// //////////////////// +static inline uint32 +decompression_iterator_items_seen(const DictionaryDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->bitmap.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const DictionaryDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->bitmap.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const DictionaryDecompressionIterator *iter) +{ + return iter->bitmap.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const DictionaryDecompressionIterator *iter) +{ + return iter->bitmap.num_elements; +} + static void dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, const char *_data, bool scan_forward, Oid element_type) @@ -470,11 +508,11 @@ dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, co for (uint32 i = 0; i < bitmap->num_distinct; i++) { DecompressResult res = array_decompression_iterator_try_next_forward(dictionary_iterator); - Assert(!res.is_null); - Assert(!res.is_done); + CheckCompressedData(!res.is_null); + CheckCompressedData(!res.is_done); iter->values[i] = res.val; } - Assert(array_decompression_iterator_try_next_forward(dictionary_iterator).is_done); + CheckCompressedData(array_decompression_iterator_try_next_forward(dictionary_iterator).is_done); } static ArrowArray *tsl_bool_dictionary_decompress_all(Datum compressed, Oid element_type, @@ -719,7 +757,8 @@ tsl_text_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryCon bool have_incorrect_index = false; for (uint32 i = 0; i < n_notnull; i++) { - have_incorrect_index = have_incorrect_index || indices[i] >= (int16) header->num_distinct; + have_incorrect_index = + have_incorrect_index || indices[i] >= (int16) header->num_distinct || indices[i] < 0; } CheckCompressedData(!have_incorrect_index); @@ -831,6 +870,12 @@ dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter_b simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -847,6 +892,12 @@ dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter_b result = simple8brle_decompression_iterator_try_next_forward(&iter->bitmap); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -876,6 +927,12 @@ dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter_b simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -892,12 +949,18 @@ dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter_b result = simple8brle_decompression_iterator_try_next_reverse(&iter->bitmap); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; } - Assert(result.val < iter->compressed->num_distinct); + CheckCompressedData(result.val < iter->compressed->num_distinct); return (DecompressResult){ .val = iter->values[result.val], .is_null = false, diff --git a/tsl/src/compression/algorithms/gorilla.c b/tsl/src/compression/algorithms/gorilla.c index 164cd212acc..e5808cc1993 100644 --- a/tsl/src/compression/algorithms/gorilla.c +++ b/tsl/src/compression/algorithms/gorilla.c @@ -122,6 +122,11 @@ typedef struct GorillaDecompressionIterator uint8 prev_leading_zeroes; uint8 prev_xor_bits_used; bool has_nulls; + + /* for validating the last_value set in the header + * and cross check with the iterator */ + uint64 last_value_to_return; + uint64 last_returned_value; } GorillaDecompressionIterator; /******************** @@ -519,6 +524,44 @@ gorilla_compressor_finish(GorillaCompressor *compressor) *** DecompressionIterator *** *******************************/ +static inline uint32 +decompression_iterator_items_seen(const GorillaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->tag0s.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const GorillaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->tag0s.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const GorillaDecompressionIterator *iter) +{ + return iter->tag0s.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const GorillaDecompressionIterator *iter) +{ + return iter->tag0s.num_elements; +} + inline static void bytes_attach_bit_array_and_advance(BitArray *dst, StringInfo si, uint32 num_buckets, uint8 bits_in_last_bucket) @@ -560,11 +603,29 @@ compressed_gorilla_data_init_from_stringinfo(CompressedGorillaData *expanded, St if (has_nulls) { expanded->nulls = bytes_deserialize_simple8b_and_advance(si); + CheckCompressedData(expanded->nulls->num_elements >= + expanded->num_bits_used_per_xor->num_elements); + CheckCompressedData(expanded->nulls->num_elements >= expanded->tag0s->num_elements); + CheckCompressedData(expanded->nulls->num_elements >= expanded->tag1s->num_elements); } else { expanded->nulls = NULL; } + + /* XOR bit count must be reasonable */ + uint64 xor_bit_count = bit_array_num_bits(&expanded->xors); + CheckCompressedData(xor_bit_count <= expanded->tag0s->num_elements * 64); + + /* leading zeros have a fixed size and we need at least one item */ + uint64 num_leading_zeros_bits = bit_array_num_bits(&expanded->leading_zeros); + CheckCompressedData(num_leading_zeros_bits >= BITS_PER_LEADING_ZEROS && + num_leading_zeros_bits % BITS_PER_LEADING_ZEROS == 0 && + num_leading_zeros_bits <= + expanded->tag0s->num_elements * BITS_PER_LEADING_ZEROS); + + /* tag bits must be reasonable too */ + CheckCompressedData(expanded->tag0s->num_elements >= expanded->tag1s->num_elements); } static void @@ -672,6 +733,14 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte /* Could slightly improve performance here by not returning a tail of non-null bits */ if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last_value in the header must match with the last that we returned */ + CheckCompressedData(iter->last_returned_value == iter->gorilla_data.header->last_value); return (DecompressResultInternal){ .is_done = true, }; @@ -689,6 +758,14 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last_value in the header must match with the last that we returned */ + CheckCompressedData(iter->last_returned_value == iter->gorilla_data.header->last_value); CheckCompressedData(!iter->has_nulls); return (DecompressResultInternal){ .is_done = true, @@ -702,11 +779,15 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte }; } + /* tag0 is a single bit, if not 0 then it has to be 1 */ + CheckCompressedData(tag0.val == 1); + tag1 = simple8brle_decompression_iterator_try_next_forward(&iter->tag1s); CheckCompressedData(!tag1.is_done); if (tag1.val != 0) { + CheckCompressedData(tag1.val == 1); Simple8bRleDecompressResult num_xor_bits; /* get new xor sizes */ iter->prev_leading_zeroes = @@ -736,6 +817,7 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte xor = bit_array_iter_next(&iter->xors, iter->prev_xor_bits_used); xor <<= 64 - (iter->prev_leading_zeroes + iter->prev_xor_bits_used); iter->prev_val ^= xor; + iter->last_returned_value = iter->prev_val; return (DecompressResultInternal){ .val = iter->prev_val, @@ -799,9 +881,34 @@ gorilla_decompression_iterator_from_datum_reverse(Datum gorilla_compressed, Oid iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); - Assert(!num_xor_bits.is_done); + CheckCompressedData(!num_xor_bits.is_done); iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_val = iter->gorilla_data.header->last_value; + + /* on reverse iteration the `last_value` in the header is critical for + * the rest of the decoding. when we receive malformed data we can only + * cross reference the result with the first value + */ + { + GorillaDecompressionIterator forward_iter; + forward_iter.base.compression_algorithm = COMPRESSION_ALGORITHM_GORILLA; + forward_iter.base.forward = true; + forward_iter.base.element_type = element_type; + forward_iter.base.try_next = gorilla_decompression_iterator_try_next_forward; + compressed_gorilla_data_init_from_datum(&forward_iter.gorilla_data, gorilla_compressed); + gorilla_iterator_init_from_expanded_forward(&forward_iter, element_type); + /* find the first non-null value */ + DecompressResultInternal first_val = + gorilla_decompression_iterator_try_next_forward_internal(&forward_iter); + int32 n = 0; + while (first_val.is_done == false && first_val.is_null) + { + CheckCompressedData(n++ < GLOBAL_MAX_ROWS_PER_COMPRESSION); + first_val = gorilla_decompression_iterator_try_next_forward_internal(&forward_iter); + } + CheckCompressedData(!first_val.is_done); + iter->last_value_to_return = first_val.val; + } return &iter->base; } @@ -820,6 +927,14 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -839,6 +954,14 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -851,6 +974,16 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte }; } + /* tag0 is a single bit, if not 0 then it has to be 1 */ + CheckCompressedData(tag0.val == 1); + + /* make sure we are in a valid state */ + tag1 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag1s); + CheckCompressedData(!tag1.is_done); + + /* check that we have this much to read */ + CheckCompressedData(iter->prev_leading_zeroes + iter->prev_xor_bits_used > 0); + CheckCompressedData(bit_array_iter_position(&iter->xors) >= iter->prev_xor_bits_used); xor = bit_array_iter_next_rev(&iter->xors, iter->prev_xor_bits_used); if (iter->prev_leading_zeroes + iter->prev_xor_bits_used < 64) @@ -859,10 +992,9 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte } iter->prev_val ^= xor; - tag1 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag1s); - if (tag1.val != 0) { + CheckCompressedData(tag1.val == 1); /* get new xor sizes */ Simple8bRleDecompressResult num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); @@ -875,12 +1007,19 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte } else { + /* make sure we have enough bits to read for the leading zeros */ + CheckCompressedData(bit_array_iter_position(&iter->leading_zeros) >= + BITS_PER_LEADING_ZEROS); iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); + + /* more than 64 bits of data doesn't make sense */ + CheckCompressedData(iter->prev_leading_zeroes + iter->prev_xor_bits_used <= 64); } } + iter->last_returned_value = val; return (DecompressResultInternal){ .val = val, }; diff --git a/tsl/src/compression/algorithms/gorilla_impl.c b/tsl/src/compression/algorithms/gorilla_impl.c index 85f6b6f512d..caa27662d86 100644 --- a/tsl/src/compression/algorithms/gorilla_impl.c +++ b/tsl/src/compression/algorithms/gorilla_impl.c @@ -9,6 +9,7 @@ * Specialized for each supported data type. */ +#include #define FUNCTION_NAME_HELPER(X, Y) X##_##Y #define FUNCTION_NAME(X, Y) FUNCTION_NAME_HELPER(X, Y) @@ -129,6 +130,7 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril } uint64 *restrict validity_bitmap = NULL; + ELEMENT_TYPE last_value = decompressed_values[n_total - 1]; if (has_nulls) { /* @@ -160,6 +162,8 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; + last_value = decompressed_values[current_notnull_element]; + for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); @@ -179,6 +183,9 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril Assert(current_notnull_element == -1); } + /* the last value in the header must match with the last that we returned */ + CheckCompressedData(last_value == (ELEMENT_TYPE) gorilla_data->header->last_value); + /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; diff --git a/tsl/src/compression/algorithms/simple8b_rle.h b/tsl/src/compression/algorithms/simple8b_rle.h index e139ae1ce90..310449ad0d5 100644 --- a/tsl/src/compression/algorithms/simple8b_rle.h +++ b/tsl/src/compression/algorithms/simple8b_rle.h @@ -6,6 +6,7 @@ #pragma once #include +#include "compression/compression.h" #include #include #include @@ -1007,12 +1008,12 @@ simple8brle_decompression_iterator_max_elements(Simple8bRleDecompressionIterator if (simple8brle_selector_is_rle(selector) && iter->compressed_data) { - Assert(simple8brle_rledata_repeatcount(iter->compressed_data[i]) > 0); + CheckCompressedData(simple8brle_rledata_repeatcount(iter->compressed_data[i]) > 0); max_stored += simple8brle_rledata_repeatcount(iter->compressed_data[i]); } else { - Assert(selector < SIMPLE8B_MAXCODE); + CheckCompressedData(selector < SIMPLE8B_MAXCODE); max_stored += SIMPLE8B_NUM_ELEMENTS[selector]; } } @@ -1029,6 +1030,7 @@ simple8brle_decompression_iterator_init_reverse(Simple8bRleDecompressionIterator skipped_in_last = simple8brle_decompression_iterator_max_elements(iter, compressed) - compressed->num_elements; + CheckCompressedData(skipped_in_last >= 0); Assert(NULL != iter->compressed_data); iter->current_block = @@ -1051,6 +1053,8 @@ simple8brle_decompression_iterator_try_next_forward(Simple8bRleDecompressionIter uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) { + /* all compressed blocks have been processed */ + CheckCompressedData(iter->num_blocks == iter->current_compressed_pos); return (Simple8bRleDecompressResult){ .is_done = true, }; @@ -1087,6 +1091,8 @@ simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIter uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) { + /* the posittion goes below zero once exhausted all elements */ + CheckCompressedData(iter->current_in_compressed_pos == -1); return (Simple8bRleDecompressResult){ .is_done = true, }; @@ -1094,10 +1100,14 @@ simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIter if (iter->current_in_compressed_pos < 0) { + CheckCompressedData(iter->current_compressed_pos >= 0); iter->current_block = simple8brle_block_create(bit_array_iter_next_rev(&iter->selectors, SIMPLE8B_BITS_PER_SELECTOR), iter->compressed_data[iter->current_compressed_pos]); + CheckCompressedData(iter->current_block.selector != 0); + CheckCompressedData(iter->current_block.num_elements_compressed <= + GLOBAL_MAX_ROWS_PER_COMPRESSION); iter->current_in_compressed_pos = iter->current_block.num_elements_compressed - 1; iter->current_compressed_pos -= 1; } diff --git a/tsl/src/compression/algorithms/uuid_compress.c b/tsl/src/compression/algorithms/uuid_compress.c index 69bc37b9985..cbf9e73374f 100644 --- a/tsl/src/compression/algorithms/uuid_compress.c +++ b/tsl/src/compression/algorithms/uuid_compress.c @@ -662,12 +662,15 @@ uuid_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { if (arrow_row_is_valid(validity_bitmap, i)) { - Assert(value_position < num_values); + CheckCompressedData(value_position < num_values); uuid_buffer[i].components[0] = pg_ntoh64(timestamp_values[i]); uuid_buffer[i].components[1] = rand_b_and_variant[value_position]; ++value_position; } } + + /* check that we returned all values */ + CheckCompressedData(value_position == num_values); } else { diff --git a/tsl/test/expected/compression_algos.out b/tsl/test/expected/compression_algos.out index c6914801097..d7287b7d9b4 100644 --- a/tsl/test/expected/compression_algos.out +++ b/tsl/test/expected/compression_algos.out @@ -2151,10 +2151,10 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 144 | XX001 | true - 93 | XX001 | XX001 - 54 | true | true + 315 | XX001 | XX001 24 | 08P01 | 08P01 + 1 | false | false + 1 | true | true \set algo deltadelta \set type int8 @@ -2166,10 +2166,9 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 108 | XX001 | XX001 - 69 | true | true - 68 | XX001 | true - 13 | 08P01 | 08P01 + 259 | XX001 | XX001 + 14 | 08P01 | 08P01 + 2 | true | true 1 | false | false \set algo array @@ -2199,13 +2198,14 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 84 | XX001 | XX001 - 13 | XX001 | true - 5 | 08P01 | 08P01 + 106 | XX001 | XX001 + 6 | 08P01 | 08P01 + 6 | 22021 | 22021 5 | true | true - 4 | 22021 | 22021 - 1 | 3F000 | 3F000 + 1 | XX001 | true 1 | false | false + 1 | true | XX001 + 1 | 3F000 | 3F000 \set algo bool \set type bool @@ -2231,8 +2231,7 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 49 | XX001 | XX001 + 50 | XX001 | XX001 2 | 08P01 | 08P01 - 1 | true | XX001 1 | true | true diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa b/tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa new file mode 100644 index 0000000000000000000000000000000000000000..65c6ef038fa0707a5639b6ea722b49e993765991 GIT binary patch literal 66 zcmZQ!wEnNapuiBpz!1d%1JXbSBLjn%J_F-7MUX55Z#qy&=OaX%fq{vEAs(nfAIN6_ E0P^||@S=ea0V5yC8jjF*zHQWBNUYfo%SFYr}qy+r^kZ`b&tl8hHx;8Ygs}+?b;b81)2#hYXD=?jSTcp@af0qh7iCq zyB~l!=612i{U<#_D=>yW&TcnDWRj6c#Ch#66XvW=O_n7E(5e2zvi6IN9b*PL2dlYU z-sRH%q`<)rI{~}Avn&?RuQ;VRdqyDZKM(*Z5g=yyH*;%jXlUq0&j0oQ>jglUJw9fWEE-v literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa b/tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa new file mode 100644 index 0000000000000000000000000000000000000000..e8990da22504cf41b1d02a125120ab985bb2b15e GIT binary patch literal 58 zcmZQ!NM?Y5|BN8oY6g%{1Y+j@|G!<(XZWVbc!z~kfm4AY9VqypQ9(h0K|z7#I|~B< DHsTAR literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 b/tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 new file mode 100644 index 0000000000000000000000000000000000000000..35d144c4d22db30ed57df4b9838e74fde04ae984 GIT binary patch literal 98 zcmZQ!V9;S~*gWIqKL#+cP5@FWK+NHW01XTb|GzRY0!3lK7|3AcumUP+?Wpf)1Ijfp dB>w;ZpMl}`e|?7X1!@2PGj%z1GiZ7o1OOhS8V~>g literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 b/tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 new file mode 100644 index 0000000000000000000000000000000000000000..eb0a4721c8f12a6665f2c69c2cea2354322ccbaf GIT binary patch literal 162 zcmZQ!U|?Wu*gWI?dmsY{tY!d-SwJi>bLNcBj*2sa3=EGL85obAxWVxMKeG!H>%k)o ztPK1!bK1b5udM?JKsp&1ZoL1=0G3f{Yhz?!Pyi}mU}9o12N`(-h!u{bf&XCs1t0)A V;{SgI1h~e+%D}*6WK_c72>_n!I^_TW literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 b/tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 new file mode 100644 index 0000000000000000000000000000000000000000..dea40ac5852c46e61853b22c861dfe8f46fcf009 GIT binary patch literal 98 zcmZQ!U|?Wu*f@jXJp%|aSj_+u8bHkP|Nq}zyPRD$U0rwWVtBU=E7<=Z9SAb~*8_=h N0*&~;<^O+1768-{HwFLz literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 b/tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 new file mode 100644 index 0000000000000000000000000000000000000000..5507b0554b8b0128dfc02982455adf2805c8c31b GIT binary patch literal 642 zcmZQ!U|?Wu*gWI?dj@78lfh~R0|QeTkm2Z+2L%%+PJ}Umv^EkOomp2`2NJ5=j0-sX z;F4hlNq`)H2Q&a(`W@^_kRp&vnZE%UEI`cIuvY@;h3eT(|9=3vDG&f;VF75?1Sv4^ z0#$M7IZg^B2oxEZ@vC7o?b8r0GRt6MmQjr8Y+^hLa)5OL1B(=^0E-pFe+DatHB<#a z&z{(@a!=NF_!l1*^cj-;RPX8I4|NmoPU|<4bMj!?OW(dWg1*BDg7$y#;8QIv_0M?vg|NjGN1_vMp0uU1f7#R$}G_yT} hC)kkx|5+IIng9QP`v3kyp!rTDLCzR~I0@ug3aZvUK$yzJFU`-1`;%u<>~ehC|3WfCRtl328z%Wf$~}x7qX#^eK9C`3U?1iXm4U literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 b/tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 new file mode 100644 index 0000000000000000000000000000000000000000..e6525757bd3854a01a07f2791680551fa915ab12 GIT binary patch literal 114 zcmZQ!U}FG*62?T9AO!|qb|9PiDUjfDXZQ~WAXyLtNF_1oGiU(mhRrkHa{&d41sKF< p0Qm|G3@Sj1AxWPBEDut?AIN_y6V2GLnZXLIhLM5k00RR@GyqOc9H;;Q literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 b/tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 new file mode 100644 index 0000000000000000000000000000000000000000..4ac1a52bbc7001e517a36ed9b83bc82ee804d057 GIT binary patch literal 193 zcmZQ#EJ%+}PAo~x$xmk}Nv$YhU|?YU0wREz1Bm$<800?yX&_*J1tbInpa1{=QE-*u zs#RbNq#(c%NP$gI1gTvGG+`CPs#OGx23ZBN31k=0CZJUU3=Ayl4E#VVrGR|qR|0|z bf(#6TK+xO6`2ROZX*mN!W{DPv1=I@wx5+Jz literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 b/tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 new file mode 100644 index 0000000000000000000000000000000000000000..74881b95264f0b4e5d71164010296ee9a6f6a56d GIT binary patch literal 49 ucmZQ#C`gY_PAo~x$xmk}Nv$YhU|?W=0VEiI|Ns9VNEDBlxASaEYSiAfs_LPC66-Z literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 b/tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 new file mode 100644 index 0000000000000000000000000000000000000000..9abbed127369fff8911efbf4c05649d69ecc029f GIT binary patch literal 73 vcmZQ#EJ%+}PAo~x$xmk}Nv$Yh00KrZRs}H`7?H#!kj25W?EkgFG$R85>+%W? literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 b/tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 new file mode 100644 index 0000000000000000000000000000000000000000..77e2469cd0f32f792852109b67d0ffe5f5ea031c GIT binary patch literal 50 pcmZQ#C`gY_PAo~x$xmk}Nv$Yh00KrR2C=ytKx7z*00Krv1_1oe2EG6Q literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab b/tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab new file mode 100644 index 0000000000000000000000000000000000000000..ea4b775d573e1c841ea10490219daae59f6b1539 GIT binary patch literal 58 ycmZQ#C`gY_PAo~x$xmk}Nv$YhU|?YU0wNd~a&jln$OADzKpQHa1D2Y{zyJW7;|py7 literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 b/tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 new file mode 100644 index 0000000000000000000000000000000000000000..21efd0e8d32f3f54b441af797b294ebf9ef8f9ed GIT binary patch literal 145 zcmZQ#EJ%+}PAo~x$xmk}Nv$YhU|?YU0wRDI2=w_G7~~zmOa{gmK$=Ss3RV>%0fs=R edWcE}hEpfpGVu9%4dY&}dD&{hC;T_S$~ literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 b/tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 new file mode 100644 index 0000000000000000000000000000000000000000..6760c9f2a0b332cc217e7ee07227ad3f57780d14 GIT binary patch literal 97 zcmZQ#EJ%+}PAo~x$xmk}Nv$Yh00KrZRs}J^Yz7AQ{~-1YAZB6U5EQIoU=Rd?RS*Ca X1?gr?EJ;tyXUGAm{Kg;yqJX*qkHHdB literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc b/tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc new file mode 100644 index 0000000000000000000000000000000000000000..17f29d8e4115f26020dfeb6e254cb24cc4cfb722 GIT binary patch literal 1854 zcmc&#J5Izf5FKZ;Wa$W0R1^pz?H!Opfw%!JtPolX5ELmm0}c1c$v6NKE%VL9CKg(; zh>A>*~jH5seE;U@qK3CytoE|IbjT3P5QLj!>9rA`Xw&vGCE@cXgP@tZ7{ zSkZ8Ym#n`7xazl9>ZGwSsXJL|t1 z3q@;aE_JJwR;s_~>-cAP(0l;UHCr)R_7zV_40>mVxf#}TrhrCM+#p7(?f|ogqFi<#_4tT1h zW5=gyRxORmGQBH^GNPg&`wdfQnYV_bBTp7)Bq%1uq=GwS%`KOa$$48QM!PB`ie|#% z;2@#U9F&(#m!a%aS@^R+`jQx@yHQ45_S2Y_N=~;#~N&E`?UhBFL(xkA<|LWb>|lBd$UB_z@pV;2_1-Dz|8@uld^9Nj&kLYCN=@Hr-^aaNhJ`{PbnvB(6(biKF sv$2N@QY^2Iv?15qKB2Rny&s&JMvAZ9m7)}^Ie*^fV2?34+T#{B?}-P1LjV8( literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob new file mode 100644 index 0000000000000000000000000000000000000000..9f41c3ead3b87ce2cae195bc341f9dacdcea3bab GIT binary patch literal 170 zcmZQ#C`gY_PAo~x$xmk}Nv$YhU|?Vt1QG%-1R-G6s#VU!01aXkp{N0xi~@kR0)YdV JL?f6O7yv3YD&_zH literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx new file mode 100644 index 0000000000000000000000000000000000000000..3073085a28b6a502fc016d54f6383d8da03e0568 GIT binary patch literal 53 ocmZQ#C`gY_PAo~x$xmk}Nv$Yh00KrR2C=z81Or0@LX@!r014v;;Q#;t literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 b/tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 new file mode 100644 index 0000000000000000000000000000000000000000..e37e5c05eed7542b99ccac055fd053feaf500a84 GIT binary patch literal 124 zcmXYpOA3H63`AdS1s6hb?MCq)QhG9v+e;Lj7z6VOGfc8!mH&3B33+zCsPdaI3C1ao d+%>shzX+P1D8VA14^ec&bcrvc)dKee!(J7N1!4dI literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 b/tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 new file mode 100644 index 0000000000000000000000000000000000000000..f155d68c6cba65292f8bb09e5deb5d943b717872 GIT binary patch literal 124 zcmZQ(WMFh)00Ji?a7#IW~;*31kfFcY|44;7{SP_T- Qg33St|NmuR(g1Ue0j{nGumAu6 literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e b/tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e new file mode 100644 index 0000000000000000000000000000000000000000..c0aa9982ec1bf761c9c57ec8ec0c9ddbe4ab18d9 GIT binary patch literal 100 zcmZQ(aA08IV1NLAAjJq}gXkZe3{3xlG&4l}8%PQ$E&!szVmu%g5I6xb6A;5R*F#w# HMGQOu_|F7R literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 b/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 new file mode 100644 index 00000000000..c471733217f --- /dev/null +++ b/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 @@ -0,0 +1 @@ +� \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 b/tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 new file mode 100644 index 0000000000000000000000000000000000000000..840e0d4427ea6f444602ac5fa1b3a8db8c54c3ed GIT binary patch literal 100 zcmZQ(aA-Whz!1yGz`(%#0!V-uAi&SSVAtRRW}=Hj1c4+Y517fo-~^bj_3;>#<1z7+9 literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 b/tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 new file mode 100644 index 0000000000000000000000000000000000000000..ed477d73ee37d73e9b1ed4b8cd3c46dbf3657530 GIT binary patch literal 124 zcmZQ({K@FR00qoInh}IqA+!UK4H9Dli2*SPz-X{q9*7u&6Oe`~0#OVs3<_Y4U}>NZ LkYY5Qh7A7!(7OXv literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 b/tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 new file mode 100644 index 0000000000000000000000000000000000000000..512250baf963dbb3a9822bbc1472ba6eb0a9ffd4 GIT binary patch literal 100 zcmZQ(aByjGU;qQ=7eI;;!UoaIAf`JID*-W*xD-enB+1AFVgZ2@5HkTWMA3IhA3iS-~d7v K1_g*Juq*(4xdHtE literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb b/tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb new file mode 100644 index 0000000000000000000000000000000000000000..ec84ad2685bbdf3f2e3d580d2b85f468da052c45 GIT binary patch literal 100 zcmZQ(fB+^Cuz}bN F3;>ei13&-( literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 b/tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 new file mode 100644 index 0000000000000000000000000000000000000000..9d6787a642e3e7dbe01431fe8c048de9d377189c GIT binary patch literal 116 zcmZQ(aByj0fPxP|n)&}91_p%&L3suS9;h%QkgiMw68r`a9~c1X<3J1){8!Jw@LveX f1_5RuW`yZ*0& literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 b/tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 new file mode 100644 index 0000000000000000000000000000000000000000..4b539c6e1dbfce14dea7a8647f2567d71b34aa48 GIT binary patch literal 116 zcmZQ(VEDiP|2GCOU;>hi5GI6XXJB9k@`cdFoq!TB0MR4>=CUv_`~@1o`~t{hX8g$r RO3RM6A literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 b/tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 new file mode 100644 index 0000000000000000000000000000000000000000..1aecf3d8fbc150e4fffef7e1e9c0d9203579da56 GIT binary patch literal 116 zcmZQ(WN36?U|?VbVg?o%gB2b KXe7uWhF$>Mj|D&g literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a b/tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a new file mode 100644 index 0000000000000000000000000000000000000000..1a321f8fa9edb42acd1a2eeccac57a64530757ec GIT binary patch literal 124 zcmZQ(WIzBTAQQ}IWnhDeGXuq#K;l{ub&L#LUkq4-%fq~)RX&?uz2rR|G!2A=4 X8T6q%AY}o>5R(gmG&5s^15f|}-U$Uk literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 b/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 new file mode 100644 index 00000000000..45a8ca02bfc --- /dev/null +++ b/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c b/tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c new file mode 100644 index 0000000000000000000000000000000000000000..0fe32f088e416f584db1cc4561badd4ec81d0262 GIT binary patch literal 116 zcmZQ(WN36?U|?VbVg?o%gB2Br}>MBR^P?5=#qvyZbTo-<(H{QK|h>O`Ozzrn)?20(fy y5HtM(56C&=GYbRA6b@#n wRtEbAKo*za|NjPpK=2<9{?#)uz&Z9HSN{Lcz{Cj={0EeQ8Gx0BxE?A60AegUQ2+n{ literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 b/tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 new file mode 100644 index 0000000000000000000000000000000000000000..edcf816253b102d2186327566d3c7187899357dc GIT binary patch literal 151 zcmZQ(aA-Wj00GP|fHWhN!_UCL1m-FNNn4O0GZ1Hk7>qn15(u1t7@~-QfrEpCnT5fD zfk6SR5Gd27v+0zAL9?&IW=@4>AfJIj>_7&PbOB;k1`eQsVEaG@00GdxM|KThCXfIC DavcpM literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 b/tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 new file mode 100644 index 0000000000000000000000000000000000000000..79dff22ef0f48517fe1d98ddfb41568d0face176 GIT binary patch literal 468 zcmZQ(aByj0fPxP|TAuEJ1!TJdgP=SE1B?APARFWiWI+cYpK-2){p{nciRTQMH~;=S zyE+jl#&7WOfdP=73B=4m3xZlR&@qeIz#bdB;jmHy!vfF5FC Vuk|YRJ1vR3R$RTlqPy9b$#bH{|tb%0RAQN-HR#AiG%l zs)aTN4E507xNl6D-v2;7&ar=c5GCx1A3q$ZV25!A+!@>DDqzj`AmUYpTq2}FjUA*c zh-pD(i)=&-GvB|Wgr)?Q$G&qDCSLMR)?7`h3ZY5Url|ODJoC4QN@ cvyzfYf~c2>`NNBV>cb-Wif^nfcwFrUp7s4bKL7v# literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b b/tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b new file mode 100644 index 0000000000000000000000000000000000000000..1553d3b0fda33e039fb96203fe90b89812675720 GIT binary patch literal 1084 zcmc&yJxfDD5S+`4#qSe;_gGv`c3z_yeMKFklm6 z=PyVpR>npY^*OtDPo5@P*f?<6y=8aa>|6qrDpfcvtuY?#D^7n0Fb)_Q+tt(&{OQ+V zUgwmJS-@j*o@#?JB?X=xo?c+JfvF&g zX$s7|V1OTt3wMH*i%Gz(hv9==ZQQCYK@l;?7p zbucJMlIDhrIQq>HMW)5&qtv>WMBV(m`2I0J#hpE-7+I9<)Ci3*<#4f<&lb;WVG&ou z9bzXxnqe5!JGAm7Rg?$Xy=H`EaycOe{U~~jchmYkLb-F7L5L~#b4lPqzVyx~Djl_b zcW}qVkuQHI`N2~^?il>MnlJNnRp5<$TGzKFf8;g10M_y@$jn^lZ0opqVYUj5PUSXS ru{E0OysdY5IkB~yLZfR}>)ZA+YwJ2T-QHxZ@R(64Wco3=+>rYK0Rd%_ literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob b/tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob new file mode 100644 index 0000000000000000000000000000000000000000..d823b9f3a137188acf9686513e6fdd23dd57aefd GIT binary patch literal 92 zcmZQ(_{r$N00q)Onh}T{SRpi!0s|%>2@(eZ7|jggm;y1f(^b literal 0 HcmV?d00001 diff --git a/tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef b/tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef new file mode 100644 index 0000000000000000000000000000000000000000..fd4c9be1d48586545413ad4c3cb6bb2611cbabb3 GIT binary patch literal 260 zcmZQ(KmexqKsFPU|DS<@MF7ZVWMI&G02Olt(gGmh0|(3u4vhz((#$V_44`@j215o0 zeg+1+1_qYDKt5QUg@NG!ND2rDsbgUH{Qp1Deny7+{|Z3$j35FCBq0<73(zcxnG%P2 x6!v!MY-03`opi$G5{IEevx4ttPK9QWN)E9D9ze`1&Kp|0*t}H4gjRAA{zhz literal 0 HcmV?d00001 diff --git a/tsl/test/src/decompress_arithmetic_test_impl.c b/tsl/test/src/decompress_arithmetic_test_impl.c index f6e898dbb5a..90dc43c83be 100644 --- a/tsl/test/src/decompress_arithmetic_test_impl.c +++ b/tsl/test/src/decompress_arithmetic_test_impl.c @@ -4,6 +4,7 @@ * LICENSE-TIMESCALE for a copy of the license. */ +#include "compression/algorithms/uuid_compress.h" #define FUNCTION_NAME_HELPER3(X, Y, Z) X##_##Y##_##Z #define FUNCTION_NAME3(X, Y, Z) FUNCTION_NAME_HELPER3(X, Y, Z) #define FUNCTION_NAME_HELPER2(X, Y) X##_##Y @@ -25,6 +26,32 @@ #define ARROW_GET_VALUE(A, I) ((CTYPE *) (A)->buffers[1])[I] #endif +static void +FUNCTION_NAME3(compare_results, CTYPE, ALGO)(DecompressResult *fwd, DecompressResult *rev, int n) +{ + for (int i = 0; i < n; i++) + { + if (fwd[i].is_null != rev[i].is_null) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("Forwards is_null %d, reverse is_null %d at row %d.", + fwd[i].is_null, + rev[i].is_null, + i))); + } + + if (!fwd[i].is_null && IS_NOT_EQUAL(DATUM_TO_CTYPE(fwd[i].val), DATUM_TO_CTYPE(rev[i].val))) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + } +} + static void FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(ArrowArray *arrow, int error_type, DecompressResult *results, int n) @@ -80,10 +107,9 @@ FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(ArrowArray *arrow, int error_type, } /* - * Try to decompress the given compressed data. Used for fuzzing and for checking - * the examples found by fuzzing. For fuzzing we do less checks to keep it - * faster and the coverage space smaller. This is a generic implementation - * for arithmetic types. + * Try to decompress the given compressed data and compare the results + * between the bulk, forward and reverse iterator. This is a generic + * implementation for arithmetic types. */ static int FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, bool bulk) @@ -106,7 +132,6 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, const CompressionAlgorithmDefinition *def = algorithm_definition(data_algo); Datum compressed_data = def->compressed_data_recv(&si); - DecompressAllFunction decompress_all = tsl_get_decompress_all_function(data_algo, PG_TYPE_OID); ArrowArray *arrow = NULL; @@ -124,6 +149,8 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, */ DecompressionIterator *iter = def->iterator_init_forward(compressed_data, PG_TYPE_OID); DecompressResult results[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results, 0xFE, sizeof(results)); + int n = 0; for (DecompressResult r = iter->try_next(iter); !r.is_done; r = iter->try_next(iter)) { @@ -135,12 +162,31 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, results[n++] = r; } - /* Check that both ways of decompression match. */ + /* + * Check that the reverse iterator also works and gives the same result + */ + DecompressionIterator *rev_iter = def->iterator_init_reverse(compressed_data, PG_TYPE_OID); + DecompressResult results_rev[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results_rev, 0xEF, sizeof(results_rev)); + int rn = n - 1; + + for (DecompressResult r = rev_iter->try_next(rev_iter); !r.is_done; + r = rev_iter->try_next(rev_iter)) + { + results_rev[rn--] = r; + } + + /* Check that the iterator based decompression matches both ways. */ if (bulk) { FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(arrow, ERROR, results, n); + FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(arrow, ERROR, results_rev, n); return n; } + else + { + FUNCTION_NAME3(compare_results, CTYPE, ALGO)(results, results_rev, n); + } /* * For row-by-row decompression, check that the result is still the same diff --git a/tsl/test/src/decompress_text_test_impl.c b/tsl/test/src/decompress_text_test_impl.c index e67f9ba5011..198098fc5d0 100644 --- a/tsl/test/src/decompress_text_test_impl.c +++ b/tsl/test/src/decompress_text_test_impl.c @@ -32,6 +32,53 @@ arrow_get_str(ArrowArray *arrow, int arrow_row, const char **str) return arrow_get_str(arrow->dictionary, dict_row, str); } +static void +decompress_generic_text_compare_results(const DecompressResult *fwd, const DecompressResult *rev, + int n) +{ + for (int i = 0; i < n; i++) + { + if (fwd[i].is_null != rev[i].is_null) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("Forwards is_null %d, reverse is_null %d at row %d.", + fwd[i].is_null, + rev[i].is_null, + i))); + } + + if (!fwd[i].is_null) + { + const Datum rowbyrow_varlena = fwd[i].val; + const size_t rowbyrow_len = VARSIZE_ANY_EXHDR(DatumGetPointer(rowbyrow_varlena)); + const char *rowbyrow_cstring = VARDATA_ANY(DatumGetPointer(rowbyrow_varlena)); + + const Datum rowbyrow_rev_varlena = rev[i].val; + const size_t rowbyrow_rev_len = + VARSIZE_ANY_EXHDR(DatumGetPointer(rowbyrow_rev_varlena)); + const char *rowbyrow_rev_cstring = VARDATA_ANY(DatumGetPointer(rowbyrow_rev_varlena)); + + if (rowbyrow_len != rowbyrow_rev_len) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + + if (strncmp(rowbyrow_cstring, rowbyrow_rev_cstring, rowbyrow_len) != 0) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + } + } +} + static void decompress_generic_text_check_arrow(ArrowArray *arrow, int errorlevel, DecompressResult *results, int n) @@ -127,6 +174,8 @@ decompress_generic_text(const uint8 *Data, size_t Size, bool bulk, int requested */ DecompressionIterator *iter = def->iterator_init_forward(compressed_data, TEXTOID); DecompressResult results[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results, 0xFE, sizeof(results)); + int n = 0; for (DecompressResult r = iter->try_next(iter); !r.is_done; r = iter->try_next(iter)) { @@ -138,14 +187,34 @@ decompress_generic_text(const uint8 *Data, size_t Size, bool bulk, int requested results[n++] = r; } + /* Reverse iterator too */ + DecompressionIterator *rev_iter = def->iterator_init_reverse(compressed_data, TEXTOID); + DecompressResult results_rev[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results_rev, 0xEF, sizeof(results_rev)); + + int rn = n - 1; + for (DecompressResult r = rev_iter->try_next(rev_iter); !r.is_done; + r = rev_iter->try_next(rev_iter)) + { + results_rev[rn--] = r; + } + if (bulk) { /* * Check that the arrow decompression result matches. */ decompress_generic_text_check_arrow(arrow, ERROR, results, n); + decompress_generic_text_check_arrow(arrow, ERROR, results_rev, n); return n; } + else + { + /* + * Check that the forward and reverse row-by-row decompression results match. + */ + decompress_generic_text_compare_results(results, results_rev, n); + } /* * For row-by-row decompression, check that the result is still the same