diff --git a/.unreleased/pr_10360 b/.unreleased/pr_10360 new file mode 100644 index 00000000000..25b032d17d3 --- /dev/null +++ b/.unreleased/pr_10360 @@ -0,0 +1,2 @@ +Fixes: #10360 Decompressor crashes with malformed compressed data +Thanks: @mdisec for reporting the issues with the validation of the compressed data during decompression diff --git a/src/adts/bit_array.h b/src/adts/bit_array.h index 7125b0c7be5..03a571453b7 100644 --- a/src/adts/bit_array.h +++ b/src/adts/bit_array.h @@ -37,6 +37,7 @@ pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *i static void bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array); /* return last num_bits in forward-order (not reverse-order); must have been written as num_bits */ static uint64 bit_array_iter_next_rev(BitArrayIterator *iter, uint8 num_bits); +static inline uint64 bit_array_iter_position(const BitArrayIterator *iter); /* I/O */ static inline void bit_array_send(StringInfo buffer, const BitArray *data); diff --git a/src/adts/bit_array_impl.h b/src/adts/bit_array_impl.h index 1a166f2aa3c..8de12730b95 100644 --- a/src/adts/bit_array_impl.h +++ b/src/adts/bit_array_impl.h @@ -85,8 +85,10 @@ bit_array_num_buckets(const BitArray *array) static inline uint64 bit_array_num_bits(const BitArray *array) { - return (BITS_PER_BUCKET * (array->buckets.num_elements - UINT64CONST(1))) + - array->bits_used_in_last_bucket; + return array->buckets.num_elements == 0 ? + 0 : + (BITS_PER_BUCKET * (array->buckets.num_elements - UINT64CONST(1))) + + array->bits_used_in_last_bucket; } static inline uint64 * @@ -250,6 +252,15 @@ bit_array_iterator_init(BitArrayIterator *iter, const BitArray *array) }; } +static inline uint64 +bit_array_iter_position(const BitArrayIterator *iter) +{ + Assert(iter->current_bucket >= 0); + return iter->current_bucket < 0 ? + 0 : + (BITS_PER_BUCKET * iter->current_bucket) + iter->bits_used_in_current_bucket; +} + pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *iter, uint8 num_bits) { @@ -302,7 +313,7 @@ bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array) { *iter = (BitArrayIterator){ .array = array, - .current_bucket = array->buckets.num_elements - 1, + .current_bucket = array->buckets.num_elements == 0 ? 0 : array->buckets.num_elements - 1, .bits_used_in_current_bucket = array->bits_used_in_last_bucket, }; } diff --git a/tsl/src/compression/algorithms/deltadelta.c b/tsl/src/compression/algorithms/deltadelta.c index 10e57d1936f..12efead3be5 100644 --- a/tsl/src/compression/algorithms/deltadelta.c +++ b/tsl/src/compression/algorithms/deltadelta.c @@ -60,6 +60,11 @@ typedef struct DeltaDeltaDecompressionIterator Simple8bRleDecompressionIterator delta_deltas; Simple8bRleDecompressionIterator nulls; bool has_nulls; + + /* for validating the last_value set in the header + * and cross check with the iterator */ + uint64 last_value_to_return; + uint64 last_returned_value; } DeltaDeltaDecompressionIterator; typedef struct DeltaDeltaCompressor @@ -481,6 +486,44 @@ delta_delta_compressor_append_value(DeltaDeltaCompressor *compressor, int64 next /**********************************************************************************/ /**********************************************************************************/ +static inline uint32 +decompression_iterator_items_seen(const DeltaDeltaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->delta_deltas.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const DeltaDeltaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->delta_deltas.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const DeltaDeltaDecompressionIterator *iter) +{ + return iter->delta_deltas.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const DeltaDeltaDecompressionIterator *iter) +{ + return iter->delta_deltas.num_elements; +} + static void int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) @@ -507,6 +550,8 @@ int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, .prev_val = 0, .prev_delta = 0, .has_nulls = has_nulls, + .last_returned_value = 0, + .last_value_to_return = header->last_value, }; simple8brle_decompression_iterator_init_forward(&iter->delta_deltas, deltas); @@ -515,9 +560,13 @@ int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_forward(&iter->nulls, nulls); + CheckCompressedData(deltas->num_elements <= nulls->num_elements); } } +static DecompressResultInternal +delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompressionIterator *iter); + static void int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) @@ -529,7 +578,7 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, DeltaDeltaCompressed *header = consumeCompressedData(&si, sizeof(DeltaDeltaCompressed)); Simple8bRleSerialized *deltas = bytes_deserialize_simple8b_and_advance(&si); - Assert(header->has_nulls == 0 || header->has_nulls == 1); + CheckCompressedData(header->has_nulls == 0 || header->has_nulls == 1); *iter = (DeltaDeltaDecompressionIterator){ .base = { @@ -541,6 +590,8 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, .prev_val = header->last_value, .prev_delta = header->last_delta, .has_nulls = header->has_nulls, + .last_returned_value = 0, + .last_value_to_return = 0, }; simple8brle_decompression_iterator_init_reverse(&iter->delta_deltas, deltas); @@ -549,6 +600,31 @@ int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_reverse(&iter->nulls, nulls); + CheckCompressedData(deltas->num_elements <= nulls->num_elements); + } + + /* on reverse iteration the `last_value` in the header is critical for + * the rest of the decoding. when we receive malformed data we can only + * cross reference the result with the first value + */ + { + DeltaDeltaDecompressionIterator forward_iter; + forward_iter.base.compression_algorithm = COMPRESSION_ALGORITHM_DELTADELTA; + forward_iter.base.forward = true; + forward_iter.base.element_type = element_type; + forward_iter.base.try_next = delta_delta_decompression_iterator_try_next_forward; + int64_decompression_iterator_init_forward(&forward_iter, compressed, element_type); + /* find the first non-null value */ + DecompressResultInternal first_val = + delta_delta_decompression_iterator_try_next_forward_internal(&forward_iter); + int32 n = 0; + while (first_val.is_done == false && first_val.is_null) + { + CheckCompressedData(n++ < GLOBAL_MAX_ROWS_PER_COMPRESSION); + first_val = delta_delta_decompression_iterator_try_next_forward_internal(&forward_iter); + } + CheckCompressedData(!first_val.is_done); + iter->last_value_to_return = first_val.val; } } @@ -615,6 +691,14 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -633,6 +717,14 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -642,6 +734,7 @@ delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompres iter->prev_delta += delta_delta; iter->prev_val += iter->prev_delta; + iter->last_returned_value = iter->prev_val; return (DecompressResultInternal){ .val = iter->prev_val, @@ -712,6 +805,14 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -719,7 +820,7 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres if (result.val != 0) { - Assert(result.val == 1); + CheckCompressedData(result.val == 1); return (DecompressResultInternal){ .is_null = true, }; @@ -730,6 +831,14 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -740,6 +849,7 @@ delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompres delta_delta = zig_zag_decode(result.val); iter->prev_val -= iter->prev_delta; iter->prev_delta -= delta_delta; + iter->last_returned_value = val; return (DecompressResultInternal){ .val = val, diff --git a/tsl/src/compression/algorithms/deltadelta_impl.c b/tsl/src/compression/algorithms/deltadelta_impl.c index 78fd29fa06d..99c0f242ab3 100644 --- a/tsl/src/compression/algorithms/deltadelta_impl.c +++ b/tsl/src/compression/algorithms/deltadelta_impl.c @@ -90,6 +90,7 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory #undef INNER_LOOP_SIZE uint64 *restrict validity_bitmap = NULL; + ELEMENT_TYPE last_value = decompressed_values[n_total - 1]; if (has_nulls) { /* Now move the data to account for nulls, and fill the validity bitmap. */ @@ -117,6 +118,7 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; + last_value = decompressed_values[current_notnull_element]; for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); @@ -136,6 +138,9 @@ FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, Memory Assert(current_notnull_element == -1); } + /* the last value in the header must match with the last that we returned */ + CheckCompressedData(last_value == (ELEMENT_TYPE) header->last_value); + /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; diff --git a/tsl/src/compression/algorithms/dictionary.c b/tsl/src/compression/algorithms/dictionary.c index 571ec23aace..29228a1c65d 100644 --- a/tsl/src/compression/algorithms/dictionary.c +++ b/tsl/src/compression/algorithms/dictionary.c @@ -416,6 +416,44 @@ dictionary_compressor_finish(DictionaryCompressor *compressor) /// Decompressor /// //////////////////// +static inline uint32 +decompression_iterator_items_seen(const DictionaryDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->bitmap.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const DictionaryDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->bitmap.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const DictionaryDecompressionIterator *iter) +{ + return iter->bitmap.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const DictionaryDecompressionIterator *iter) +{ + return iter->bitmap.num_elements; +} + static void dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, const char *_data, bool scan_forward, Oid element_type) @@ -470,11 +508,11 @@ dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, co for (uint32 i = 0; i < bitmap->num_distinct; i++) { DecompressResult res = array_decompression_iterator_try_next_forward(dictionary_iterator); - Assert(!res.is_null); - Assert(!res.is_done); + CheckCompressedData(!res.is_null); + CheckCompressedData(!res.is_done); iter->values[i] = res.val; } - Assert(array_decompression_iterator_try_next_forward(dictionary_iterator).is_done); + CheckCompressedData(array_decompression_iterator_try_next_forward(dictionary_iterator).is_done); } static ArrowArray *tsl_bool_dictionary_decompress_all(Datum compressed, Oid element_type, @@ -719,7 +757,8 @@ tsl_text_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryCon bool have_incorrect_index = false; for (uint32 i = 0; i < n_notnull; i++) { - have_incorrect_index = have_incorrect_index || indices[i] >= (int16) header->num_distinct; + have_incorrect_index = + have_incorrect_index || indices[i] >= (int16) header->num_distinct || indices[i] < 0; } CheckCompressedData(!have_incorrect_index); @@ -831,6 +870,12 @@ dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter_b simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -847,6 +892,12 @@ dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter_b result = simple8brle_decompression_iterator_try_next_forward(&iter->bitmap); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -876,6 +927,12 @@ dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter_b simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; @@ -892,12 +949,18 @@ dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter_b result = simple8brle_decompression_iterator_try_next_reverse(&iter->bitmap); if (result.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); return (DecompressResult){ .is_done = true, }; } - Assert(result.val < iter->compressed->num_distinct); + CheckCompressedData(result.val < iter->compressed->num_distinct); return (DecompressResult){ .val = iter->values[result.val], .is_null = false, diff --git a/tsl/src/compression/algorithms/gorilla.c b/tsl/src/compression/algorithms/gorilla.c index 164cd212acc..e5808cc1993 100644 --- a/tsl/src/compression/algorithms/gorilla.c +++ b/tsl/src/compression/algorithms/gorilla.c @@ -122,6 +122,11 @@ typedef struct GorillaDecompressionIterator uint8 prev_leading_zeroes; uint8 prev_xor_bits_used; bool has_nulls; + + /* for validating the last_value set in the header + * and cross check with the iterator */ + uint64 last_value_to_return; + uint64 last_returned_value; } GorillaDecompressionIterator; /******************** @@ -519,6 +524,44 @@ gorilla_compressor_finish(GorillaCompressor *compressor) *** DecompressionIterator *** *******************************/ +static inline uint32 +decompression_iterator_items_seen(const GorillaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements_returned; + } + else + { + return iter->tag0s.num_elements_returned; + } +} + +static inline uint32 +decompression_iterator_item_count(const GorillaDecompressionIterator *iter) +{ + if (iter->has_nulls) + { + return iter->nulls.num_elements; + } + else + { + return iter->tag0s.num_elements; + } +} + +static inline uint32 +decompression_iterator_values_seen(const GorillaDecompressionIterator *iter) +{ + return iter->tag0s.num_elements_returned; +} + +static inline uint32 +decompression_iterator_value_count(const GorillaDecompressionIterator *iter) +{ + return iter->tag0s.num_elements; +} + inline static void bytes_attach_bit_array_and_advance(BitArray *dst, StringInfo si, uint32 num_buckets, uint8 bits_in_last_bucket) @@ -560,11 +603,29 @@ compressed_gorilla_data_init_from_stringinfo(CompressedGorillaData *expanded, St if (has_nulls) { expanded->nulls = bytes_deserialize_simple8b_and_advance(si); + CheckCompressedData(expanded->nulls->num_elements >= + expanded->num_bits_used_per_xor->num_elements); + CheckCompressedData(expanded->nulls->num_elements >= expanded->tag0s->num_elements); + CheckCompressedData(expanded->nulls->num_elements >= expanded->tag1s->num_elements); } else { expanded->nulls = NULL; } + + /* XOR bit count must be reasonable */ + uint64 xor_bit_count = bit_array_num_bits(&expanded->xors); + CheckCompressedData(xor_bit_count <= expanded->tag0s->num_elements * 64); + + /* leading zeros have a fixed size and we need at least one item */ + uint64 num_leading_zeros_bits = bit_array_num_bits(&expanded->leading_zeros); + CheckCompressedData(num_leading_zeros_bits >= BITS_PER_LEADING_ZEROS && + num_leading_zeros_bits % BITS_PER_LEADING_ZEROS == 0 && + num_leading_zeros_bits <= + expanded->tag0s->num_elements * BITS_PER_LEADING_ZEROS); + + /* tag bits must be reasonable too */ + CheckCompressedData(expanded->tag0s->num_elements >= expanded->tag1s->num_elements); } static void @@ -672,6 +733,14 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte /* Could slightly improve performance here by not returning a tail of non-null bits */ if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last_value in the header must match with the last that we returned */ + CheckCompressedData(iter->last_returned_value == iter->gorilla_data.header->last_value); return (DecompressResultInternal){ .is_done = true, }; @@ -689,6 +758,14 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last_value in the header must match with the last that we returned */ + CheckCompressedData(iter->last_returned_value == iter->gorilla_data.header->last_value); CheckCompressedData(!iter->has_nulls); return (DecompressResultInternal){ .is_done = true, @@ -702,11 +779,15 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte }; } + /* tag0 is a single bit, if not 0 then it has to be 1 */ + CheckCompressedData(tag0.val == 1); + tag1 = simple8brle_decompression_iterator_try_next_forward(&iter->tag1s); CheckCompressedData(!tag1.is_done); if (tag1.val != 0) { + CheckCompressedData(tag1.val == 1); Simple8bRleDecompressResult num_xor_bits; /* get new xor sizes */ iter->prev_leading_zeroes = @@ -736,6 +817,7 @@ gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIte xor = bit_array_iter_next(&iter->xors, iter->prev_xor_bits_used); xor <<= 64 - (iter->prev_leading_zeroes + iter->prev_xor_bits_used); iter->prev_val ^= xor; + iter->last_returned_value = iter->prev_val; return (DecompressResultInternal){ .val = iter->prev_val, @@ -799,9 +881,34 @@ gorilla_decompression_iterator_from_datum_reverse(Datum gorilla_compressed, Oid iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); - Assert(!num_xor_bits.is_done); + CheckCompressedData(!num_xor_bits.is_done); iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_val = iter->gorilla_data.header->last_value; + + /* on reverse iteration the `last_value` in the header is critical for + * the rest of the decoding. when we receive malformed data we can only + * cross reference the result with the first value + */ + { + GorillaDecompressionIterator forward_iter; + forward_iter.base.compression_algorithm = COMPRESSION_ALGORITHM_GORILLA; + forward_iter.base.forward = true; + forward_iter.base.element_type = element_type; + forward_iter.base.try_next = gorilla_decompression_iterator_try_next_forward; + compressed_gorilla_data_init_from_datum(&forward_iter.gorilla_data, gorilla_compressed); + gorilla_iterator_init_from_expanded_forward(&forward_iter, element_type); + /* find the first non-null value */ + DecompressResultInternal first_val = + gorilla_decompression_iterator_try_next_forward_internal(&forward_iter); + int32 n = 0; + while (first_val.is_done == false && first_val.is_null) + { + CheckCompressedData(n++ < GLOBAL_MAX_ROWS_PER_COMPRESSION); + first_val = gorilla_decompression_iterator_try_next_forward_internal(&forward_iter); + } + CheckCompressedData(!first_val.is_done); + iter->last_value_to_return = first_val.val; + } return &iter->base; } @@ -820,6 +927,14 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte if (null.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -839,6 +954,14 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) { + /* make sure we exhausted all items before */ + CheckCompressedData(decompression_iterator_items_seen(iter) == + decompression_iterator_item_count(iter)); + /* and also that we returned all values */ + CheckCompressedData(decompression_iterator_values_seen(iter) == + decompression_iterator_value_count(iter)); + /* the last element must match the expected value */ + CheckCompressedData(iter->last_returned_value == iter->last_value_to_return); return (DecompressResultInternal){ .is_done = true, }; @@ -851,6 +974,16 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte }; } + /* tag0 is a single bit, if not 0 then it has to be 1 */ + CheckCompressedData(tag0.val == 1); + + /* make sure we are in a valid state */ + tag1 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag1s); + CheckCompressedData(!tag1.is_done); + + /* check that we have this much to read */ + CheckCompressedData(iter->prev_leading_zeroes + iter->prev_xor_bits_used > 0); + CheckCompressedData(bit_array_iter_position(&iter->xors) >= iter->prev_xor_bits_used); xor = bit_array_iter_next_rev(&iter->xors, iter->prev_xor_bits_used); if (iter->prev_leading_zeroes + iter->prev_xor_bits_used < 64) @@ -859,10 +992,9 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte } iter->prev_val ^= xor; - tag1 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag1s); - if (tag1.val != 0) { + CheckCompressedData(tag1.val == 1); /* get new xor sizes */ Simple8bRleDecompressResult num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); @@ -875,12 +1007,19 @@ gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIte } else { + /* make sure we have enough bits to read for the leading zeros */ + CheckCompressedData(bit_array_iter_position(&iter->leading_zeros) >= + BITS_PER_LEADING_ZEROS); iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); + + /* more than 64 bits of data doesn't make sense */ + CheckCompressedData(iter->prev_leading_zeroes + iter->prev_xor_bits_used <= 64); } } + iter->last_returned_value = val; return (DecompressResultInternal){ .val = val, }; diff --git a/tsl/src/compression/algorithms/gorilla_impl.c b/tsl/src/compression/algorithms/gorilla_impl.c index 85f6b6f512d..caa27662d86 100644 --- a/tsl/src/compression/algorithms/gorilla_impl.c +++ b/tsl/src/compression/algorithms/gorilla_impl.c @@ -9,6 +9,7 @@ * Specialized for each supported data type. */ +#include #define FUNCTION_NAME_HELPER(X, Y) X##_##Y #define FUNCTION_NAME(X, Y) FUNCTION_NAME_HELPER(X, Y) @@ -129,6 +130,7 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril } uint64 *restrict validity_bitmap = NULL; + ELEMENT_TYPE last_value = decompressed_values[n_total - 1]; if (has_nulls) { /* @@ -160,6 +162,8 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; + last_value = decompressed_values[current_notnull_element]; + for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); @@ -179,6 +183,9 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril Assert(current_notnull_element == -1); } + /* the last value in the header must match with the last that we returned */ + CheckCompressedData(last_value == (ELEMENT_TYPE) gorilla_data->header->last_value); + /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; diff --git a/tsl/src/compression/algorithms/simple8b_rle.h b/tsl/src/compression/algorithms/simple8b_rle.h index e139ae1ce90..310449ad0d5 100644 --- a/tsl/src/compression/algorithms/simple8b_rle.h +++ b/tsl/src/compression/algorithms/simple8b_rle.h @@ -6,6 +6,7 @@ #pragma once #include +#include "compression/compression.h" #include #include #include @@ -1007,12 +1008,12 @@ simple8brle_decompression_iterator_max_elements(Simple8bRleDecompressionIterator if (simple8brle_selector_is_rle(selector) && iter->compressed_data) { - Assert(simple8brle_rledata_repeatcount(iter->compressed_data[i]) > 0); + CheckCompressedData(simple8brle_rledata_repeatcount(iter->compressed_data[i]) > 0); max_stored += simple8brle_rledata_repeatcount(iter->compressed_data[i]); } else { - Assert(selector < SIMPLE8B_MAXCODE); + CheckCompressedData(selector < SIMPLE8B_MAXCODE); max_stored += SIMPLE8B_NUM_ELEMENTS[selector]; } } @@ -1029,6 +1030,7 @@ simple8brle_decompression_iterator_init_reverse(Simple8bRleDecompressionIterator skipped_in_last = simple8brle_decompression_iterator_max_elements(iter, compressed) - compressed->num_elements; + CheckCompressedData(skipped_in_last >= 0); Assert(NULL != iter->compressed_data); iter->current_block = @@ -1051,6 +1053,8 @@ simple8brle_decompression_iterator_try_next_forward(Simple8bRleDecompressionIter uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) { + /* all compressed blocks have been processed */ + CheckCompressedData(iter->num_blocks == iter->current_compressed_pos); return (Simple8bRleDecompressResult){ .is_done = true, }; @@ -1087,6 +1091,8 @@ simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIter uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) { + /* the posittion goes below zero once exhausted all elements */ + CheckCompressedData(iter->current_in_compressed_pos == -1); return (Simple8bRleDecompressResult){ .is_done = true, }; @@ -1094,10 +1100,14 @@ simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIter if (iter->current_in_compressed_pos < 0) { + CheckCompressedData(iter->current_compressed_pos >= 0); iter->current_block = simple8brle_block_create(bit_array_iter_next_rev(&iter->selectors, SIMPLE8B_BITS_PER_SELECTOR), iter->compressed_data[iter->current_compressed_pos]); + CheckCompressedData(iter->current_block.selector != 0); + CheckCompressedData(iter->current_block.num_elements_compressed <= + GLOBAL_MAX_ROWS_PER_COMPRESSION); iter->current_in_compressed_pos = iter->current_block.num_elements_compressed - 1; iter->current_compressed_pos -= 1; } diff --git a/tsl/src/compression/algorithms/uuid_compress.c b/tsl/src/compression/algorithms/uuid_compress.c index 69bc37b9985..cbf9e73374f 100644 --- a/tsl/src/compression/algorithms/uuid_compress.c +++ b/tsl/src/compression/algorithms/uuid_compress.c @@ -662,12 +662,15 @@ uuid_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { if (arrow_row_is_valid(validity_bitmap, i)) { - Assert(value_position < num_values); + CheckCompressedData(value_position < num_values); uuid_buffer[i].components[0] = pg_ntoh64(timestamp_values[i]); uuid_buffer[i].components[1] = rand_b_and_variant[value_position]; ++value_position; } } + + /* check that we returned all values */ + CheckCompressedData(value_position == num_values); } else { diff --git a/tsl/test/expected/compression_algos.out b/tsl/test/expected/compression_algos.out index c6914801097..d7287b7d9b4 100644 --- a/tsl/test/expected/compression_algos.out +++ b/tsl/test/expected/compression_algos.out @@ -2151,10 +2151,10 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 144 | XX001 | true - 93 | XX001 | XX001 - 54 | true | true + 315 | XX001 | XX001 24 | 08P01 | 08P01 + 1 | false | false + 1 | true | true \set algo deltadelta \set type int8 @@ -2166,10 +2166,9 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 108 | XX001 | XX001 - 69 | true | true - 68 | XX001 | true - 13 | 08P01 | 08P01 + 259 | XX001 | XX001 + 14 | 08P01 | 08P01 + 2 | true | true 1 | false | false \set algo array @@ -2199,13 +2198,14 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 84 | XX001 | XX001 - 13 | XX001 | true - 5 | 08P01 | 08P01 + 106 | XX001 | XX001 + 6 | 08P01 | 08P01 + 6 | 22021 | 22021 5 | true | true - 4 | 22021 | 22021 - 1 | 3F000 | 3F000 + 1 | XX001 | true 1 | false | false + 1 | true | XX001 + 1 | 3F000 | 3F000 \set algo bool \set type bool @@ -2231,8 +2231,7 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 49 | XX001 | XX001 + 50 | XX001 | XX001 2 | 08P01 | 08P01 - 1 | true | XX001 1 | true | true diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa b/tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa new file mode 100644 index 00000000000..65c6ef038fa Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/00cc94b292a8e3ac0c7b36a851cbdcf4f30864aa differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/05a79f06cf3f67f726dae68d18a2290f6c9a50c9 b/tsl/test/fuzzing/compression/deltadelta-int8/05a79f06cf3f67f726dae68d18a2290f6c9a50c9 new file mode 100644 index 00000000000..22ded55aa2c --- /dev/null +++ b/tsl/test/fuzzing/compression/deltadelta-int8/05a79f06cf3f67f726dae68d18a2290f6c9a50c9 @@ -0,0 +1 @@ +: \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/1015e4b50aba50649bb467930cfe971059d6ae9d b/tsl/test/fuzzing/compression/deltadelta-int8/1015e4b50aba50649bb467930cfe971059d6ae9d new file mode 100644 index 00000000000..9278985074d Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/1015e4b50aba50649bb467930cfe971059d6ae9d differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/174db52f92ceb9f868c11cd0040ca4a20801e6fc b/tsl/test/fuzzing/compression/deltadelta-int8/174db52f92ceb9f868c11cd0040ca4a20801e6fc new file mode 100644 index 00000000000..2b519db1c64 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/174db52f92ceb9f868c11cd0040ca4a20801e6fc differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/467225894f603fcf9f2160d548ebc478bc7a4875 b/tsl/test/fuzzing/compression/deltadelta-int8/467225894f603fcf9f2160d548ebc478bc7a4875 new file mode 100644 index 00000000000..b7093e62ea2 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/467225894f603fcf9f2160d548ebc478bc7a4875 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/56527dcf9aa46bf3d1dbe57ad731b921c7eeb004 b/tsl/test/fuzzing/compression/deltadelta-int8/56527dcf9aa46bf3d1dbe57ad731b921c7eeb004 new file mode 100644 index 00000000000..7936b1977cb Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/56527dcf9aa46bf3d1dbe57ad731b921c7eeb004 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa b/tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa new file mode 100644 index 00000000000..e8990da2250 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/8d51515ae0fdb890a4232de9cd9b1461d83577fa differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 b/tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 new file mode 100644 index 00000000000..35d144c4d22 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/98561f0a5f98d0c063be2e5395136833c804b575 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 b/tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 new file mode 100644 index 00000000000..eb0a4721c8f Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/a78ca6006b321a392f21f7360a4ff7399a0d9e11 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 b/tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 new file mode 100644 index 00000000000..dea40ac5852 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/c99004415eb02cf0e1ffdf80350dcb533d94b4a5 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 b/tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 new file mode 100644 index 00000000000..5507b0554b8 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/crash-3a7aac7173e20379aa5d70ed30a1545fe0b18b73 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-581aa419c8bb6424793f216d0fda5f1470749f0d b/tsl/test/fuzzing/compression/deltadelta-int8/crash-581aa419c8bb6424793f216d0fda5f1470749f0d new file mode 100644 index 00000000000..f22aad5b298 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/crash-581aa419c8bb6424793f216d0fda5f1470749f0d differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-b666969713074b288af5d833bda5647fad29b055 b/tsl/test/fuzzing/compression/deltadelta-int8/crash-b666969713074b288af5d833bda5647fad29b055 new file mode 100644 index 00000000000..fb61f1db153 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/crash-b666969713074b288af5d833bda5647fad29b055 differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-da39a3ee5e6b4b0d3255bfef95601890afd80709 b/tsl/test/fuzzing/compression/deltadelta-int8/crash-da39a3ee5e6b4b0d3255bfef95601890afd80709 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/crash-de00eb0e8cb99709d1cf37f4b5e36b4dca7e529c b/tsl/test/fuzzing/compression/deltadelta-int8/crash-de00eb0e8cb99709d1cf37f4b5e36b4dca7e529c new file mode 100644 index 00000000000..76159691d84 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/crash-de00eb0e8cb99709d1cf37f4b5e36b4dca7e529c differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/d26928947e574611b54da6ee673f98e5bed26f3f b/tsl/test/fuzzing/compression/deltadelta-int8/d26928947e574611b54da6ee673f98e5bed26f3f new file mode 100644 index 00000000000..5e2b708f6d8 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/d26928947e574611b54da6ee673f98e5bed26f3f differ diff --git a/tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 b/tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 new file mode 100644 index 00000000000..e6525757bd3 Binary files /dev/null and b/tsl/test/fuzzing/compression/deltadelta-int8/e3c7308df09bb6c90f477547d7fa90dce9c79629 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 b/tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 new file mode 100644 index 00000000000..4ac1a52bbc7 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/007929c46768d6da44b95e8e8f62cf144be2b649 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 b/tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 new file mode 100644 index 00000000000..74881b95264 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/01a73a5c2e29161dd645c094052ebebef2ed2512 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/0371e59314a5f7e3e6525df351df7b0b2353a0d2 b/tsl/test/fuzzing/compression/dictionary-text/0371e59314a5f7e3e6525df351df7b0b2353a0d2 new file mode 100644 index 00000000000..4bfeb4e5922 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/0371e59314a5f7e3e6525df351df7b0b2353a0d2 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/096864a5697e5508063ed40b3b53f8a631910888 b/tsl/test/fuzzing/compression/dictionary-text/096864a5697e5508063ed40b3b53f8a631910888 new file mode 100644 index 00000000000..56b6863299f Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/096864a5697e5508063ed40b3b53f8a631910888 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 b/tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 new file mode 100644 index 00000000000..9abbed12736 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/098d8cee8ae94cbe21a61cc321f46c7f01cd4db2 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 b/tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 new file mode 100644 index 00000000000..77e2469cd0f Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/426effe2b9af1923459a27c89e3c4831417936a6 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab b/tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab new file mode 100644 index 00000000000..ea4b775d573 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/6b3b505c196626a582333b7762c696c031d0c6ab differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 b/tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 new file mode 100644 index 00000000000..21efd0e8d32 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/855d7a2f02334aa8ef42aee9599ce204c9725417 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 b/tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 new file mode 100644 index 00000000000..6760c9f2a0b Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/9be71e3b8c8157dfea5323c36ab1cf1c0b50b552 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc b/tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc new file mode 100644 index 00000000000..17f29d8e411 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/crash-03099ccd2c00298d4d93c6bd7e0c675c8c6e98fc differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-6fe984d76d7457879cee2fa63045473c55409ea0 b/tsl/test/fuzzing/compression/dictionary-text/crash-6fe984d76d7457879cee2fa63045473c55409ea0 new file mode 100644 index 00000000000..1d6dea095c5 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/crash-6fe984d76d7457879cee2fa63045473c55409ea0 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-70c68b0f5b1afee677c690f49a7fcfe0973614b3 b/tsl/test/fuzzing/compression/dictionary-text/crash-70c68b0f5b1afee677c690f49a7fcfe0973614b3 new file mode 100644 index 00000000000..4f64d7055a6 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/crash-70c68b0f5b1afee677c690f49a7fcfe0973614b3 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob new file mode 100644 index 00000000000..9f41c3ead3b Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-idx-values-oob differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx new file mode 100644 index 00000000000..3073085a28b Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/crash-pr10360-negative-idx differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 b/tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 new file mode 100644 index 00000000000..e37e5c05eed Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/014436ca4af1061ff896fe78c472312dd723d413 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 b/tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 new file mode 100644 index 00000000000..f155d68c6cb Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/02269e63b0856b2e0d7c3d222b61896636377bf4 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/07726c86c6954a77f1ddf28907b67229fe0c6310 b/tsl/test/fuzzing/compression/gorilla-float8/07726c86c6954a77f1ddf28907b67229fe0c6310 new file mode 100644 index 00000000000..a1be8b203b2 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/07726c86c6954a77f1ddf28907b67229fe0c6310 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e b/tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e new file mode 100644 index 00000000000..c0aa9982ec1 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/105503ff3f2307d622ce91aa7d684348abb6170e differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 b/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 new file mode 100644 index 00000000000..c471733217f --- /dev/null +++ b/tsl/test/fuzzing/compression/gorilla-float8/121a9af889bd4ca2266be5a4f680d3bead8d02d6 @@ -0,0 +1 @@ +� \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 b/tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 new file mode 100644 index 00000000000..840e0d4427e Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/1618a6d460ecedc4c8f9a946ebd2ffd6e50a4d02 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 b/tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 new file mode 100644 index 00000000000..ed477d73ee3 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/175b9e2d8cbef19f39a1bd2a4117bf4d675b23a3 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 b/tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 new file mode 100644 index 00000000000..512250baf96 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/1f070ff69fa02284c29dd02649ae54ef0f597570 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/2187f25c223185c4dc11ed35fcd308d7008eeec2 b/tsl/test/fuzzing/compression/gorilla-float8/2187f25c223185c4dc11ed35fcd308d7008eeec2 new file mode 100644 index 00000000000..3004c899f6e Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/2187f25c223185c4dc11ed35fcd308d7008eeec2 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb b/tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb new file mode 100644 index 00000000000..ec84ad2685b Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/236c0ffa09a1bfddb1efbd955589bafa3ab4a9cb differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 b/tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 new file mode 100644 index 00000000000..9d6787a642e Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/2a5f5f7a9bc26e210d02659f238a1781d0974ee5 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 b/tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 new file mode 100644 index 00000000000..4b539c6e1db Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/372bbd248990d2654a96083580591ed09ea515a2 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 b/tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 new file mode 100644 index 00000000000..1aecf3d8fbc Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/54dee5a23c3c0b34009bf11057ae923c26f50224 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a b/tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a new file mode 100644 index 00000000000..1a321f8fa9e Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/8476f031bf50af967fc23b957dcb24802f8e228a differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 b/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 new file mode 100644 index 00000000000..45a8ca02bfc --- /dev/null +++ b/tsl/test/fuzzing/compression/gorilla-float8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c b/tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c new file mode 100644 index 00000000000..0fe32f088e4 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/b470e82376caa703fcb832a140d3738d3734223c differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/c77acb44cc2f309d06b35c132bf96d0a44aedbb3 b/tsl/test/fuzzing/compression/gorilla-float8/c77acb44cc2f309d06b35c132bf96d0a44aedbb3 new file mode 100644 index 00000000000..f6524cefd5d Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/c77acb44cc2f309d06b35c132bf96d0a44aedbb3 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-0e4e2dc4fef8a8d9272ad0e2530093a2fc527196 b/tsl/test/fuzzing/compression/gorilla-float8/crash-0e4e2dc4fef8a8d9272ad0e2530093a2fc527196 new file mode 100644 index 00000000000..f818f42bdeb Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-0e4e2dc4fef8a8d9272ad0e2530093a2fc527196 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-51ea8f10ebab4dfe07d48289eaec8670472dde52 b/tsl/test/fuzzing/compression/gorilla-float8/crash-51ea8f10ebab4dfe07d48289eaec8670472dde52 new file mode 100644 index 00000000000..3e11c049aca Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-51ea8f10ebab4dfe07d48289eaec8670472dde52 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-652069006785dfb361c63ac6db2b8cbb4994510f b/tsl/test/fuzzing/compression/gorilla-float8/crash-652069006785dfb361c63ac6db2b8cbb4994510f new file mode 100644 index 00000000000..21d35062c0f Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-652069006785dfb361c63ac6db2b8cbb4994510f differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 b/tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 new file mode 100644 index 00000000000..edcf816253b Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-687a1db698c5d9b85fa19eaed1572d903d958018 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 b/tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 new file mode 100644 index 00000000000..79dff22ef0f Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-8edac9852766106ba0a087eeb07c981fc71f1524 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-9c7d5d32a7af25fd5d31a83975fcc4d734f17437 b/tsl/test/fuzzing/compression/gorilla-float8/crash-9c7d5d32a7af25fd5d31a83975fcc4d734f17437 new file mode 100644 index 00000000000..b360686ae32 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-9c7d5d32a7af25fd5d31a83975fcc4d734f17437 differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b b/tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b new file mode 100644 index 00000000000..1553d3b0fda Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-d789fb4f8fcd8f688b2dccc6c94b60ffb1fa905b differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob b/tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob new file mode 100644 index 00000000000..d823b9f3a13 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/crash-pr10360-leading-zeros-oob differ diff --git a/tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef b/tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef new file mode 100644 index 00000000000..fd4c9be1d48 Binary files /dev/null and b/tsl/test/fuzzing/compression/gorilla-float8/d5f0e700f5e3bc8393cb11dde18c07d38e910bef differ diff --git a/tsl/test/src/decompress_arithmetic_test_impl.c b/tsl/test/src/decompress_arithmetic_test_impl.c index f6e898dbb5a..90dc43c83be 100644 --- a/tsl/test/src/decompress_arithmetic_test_impl.c +++ b/tsl/test/src/decompress_arithmetic_test_impl.c @@ -4,6 +4,7 @@ * LICENSE-TIMESCALE for a copy of the license. */ +#include "compression/algorithms/uuid_compress.h" #define FUNCTION_NAME_HELPER3(X, Y, Z) X##_##Y##_##Z #define FUNCTION_NAME3(X, Y, Z) FUNCTION_NAME_HELPER3(X, Y, Z) #define FUNCTION_NAME_HELPER2(X, Y) X##_##Y @@ -25,6 +26,32 @@ #define ARROW_GET_VALUE(A, I) ((CTYPE *) (A)->buffers[1])[I] #endif +static void +FUNCTION_NAME3(compare_results, CTYPE, ALGO)(DecompressResult *fwd, DecompressResult *rev, int n) +{ + for (int i = 0; i < n; i++) + { + if (fwd[i].is_null != rev[i].is_null) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("Forwards is_null %d, reverse is_null %d at row %d.", + fwd[i].is_null, + rev[i].is_null, + i))); + } + + if (!fwd[i].is_null && IS_NOT_EQUAL(DATUM_TO_CTYPE(fwd[i].val), DATUM_TO_CTYPE(rev[i].val))) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + } +} + static void FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(ArrowArray *arrow, int error_type, DecompressResult *results, int n) @@ -80,10 +107,9 @@ FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(ArrowArray *arrow, int error_type, } /* - * Try to decompress the given compressed data. Used for fuzzing and for checking - * the examples found by fuzzing. For fuzzing we do less checks to keep it - * faster and the coverage space smaller. This is a generic implementation - * for arithmetic types. + * Try to decompress the given compressed data and compare the results + * between the bulk, forward and reverse iterator. This is a generic + * implementation for arithmetic types. */ static int FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, bool bulk) @@ -106,7 +132,6 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, const CompressionAlgorithmDefinition *def = algorithm_definition(data_algo); Datum compressed_data = def->compressed_data_recv(&si); - DecompressAllFunction decompress_all = tsl_get_decompress_all_function(data_algo, PG_TYPE_OID); ArrowArray *arrow = NULL; @@ -124,6 +149,8 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, */ DecompressionIterator *iter = def->iterator_init_forward(compressed_data, PG_TYPE_OID); DecompressResult results[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results, 0xFE, sizeof(results)); + int n = 0; for (DecompressResult r = iter->try_next(iter); !r.is_done; r = iter->try_next(iter)) { @@ -135,12 +162,31 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, results[n++] = r; } - /* Check that both ways of decompression match. */ + /* + * Check that the reverse iterator also works and gives the same result + */ + DecompressionIterator *rev_iter = def->iterator_init_reverse(compressed_data, PG_TYPE_OID); + DecompressResult results_rev[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results_rev, 0xEF, sizeof(results_rev)); + int rn = n - 1; + + for (DecompressResult r = rev_iter->try_next(rev_iter); !r.is_done; + r = rev_iter->try_next(rev_iter)) + { + results_rev[rn--] = r; + } + + /* Check that the iterator based decompression matches both ways. */ if (bulk) { FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(arrow, ERROR, results, n); + FUNCTION_NAME3(check_arrow, CTYPE, ALGO)(arrow, ERROR, results_rev, n); return n; } + else + { + FUNCTION_NAME3(compare_results, CTYPE, ALGO)(results, results_rev, n); + } /* * For row-by-row decompression, check that the result is still the same diff --git a/tsl/test/src/decompress_text_test_impl.c b/tsl/test/src/decompress_text_test_impl.c index e67f9ba5011..198098fc5d0 100644 --- a/tsl/test/src/decompress_text_test_impl.c +++ b/tsl/test/src/decompress_text_test_impl.c @@ -32,6 +32,53 @@ arrow_get_str(ArrowArray *arrow, int arrow_row, const char **str) return arrow_get_str(arrow->dictionary, dict_row, str); } +static void +decompress_generic_text_compare_results(const DecompressResult *fwd, const DecompressResult *rev, + int n) +{ + for (int i = 0; i < n; i++) + { + if (fwd[i].is_null != rev[i].is_null) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("Forwards is_null %d, reverse is_null %d at row %d.", + fwd[i].is_null, + rev[i].is_null, + i))); + } + + if (!fwd[i].is_null) + { + const Datum rowbyrow_varlena = fwd[i].val; + const size_t rowbyrow_len = VARSIZE_ANY_EXHDR(DatumGetPointer(rowbyrow_varlena)); + const char *rowbyrow_cstring = VARDATA_ANY(DatumGetPointer(rowbyrow_varlena)); + + const Datum rowbyrow_rev_varlena = rev[i].val; + const size_t rowbyrow_rev_len = + VARSIZE_ANY_EXHDR(DatumGetPointer(rowbyrow_rev_varlena)); + const char *rowbyrow_rev_cstring = VARDATA_ANY(DatumGetPointer(rowbyrow_rev_varlena)); + + if (rowbyrow_len != rowbyrow_rev_len) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + + if (strncmp(rowbyrow_cstring, rowbyrow_rev_cstring, rowbyrow_len) != 0) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("the forward and reverse decompression results do not match"), + errdetail("At row %d\n", i))); + } + } + } +} + static void decompress_generic_text_check_arrow(ArrowArray *arrow, int errorlevel, DecompressResult *results, int n) @@ -127,6 +174,8 @@ decompress_generic_text(const uint8 *Data, size_t Size, bool bulk, int requested */ DecompressionIterator *iter = def->iterator_init_forward(compressed_data, TEXTOID); DecompressResult results[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results, 0xFE, sizeof(results)); + int n = 0; for (DecompressResult r = iter->try_next(iter); !r.is_done; r = iter->try_next(iter)) { @@ -138,14 +187,34 @@ decompress_generic_text(const uint8 *Data, size_t Size, bool bulk, int requested results[n++] = r; } + /* Reverse iterator too */ + DecompressionIterator *rev_iter = def->iterator_init_reverse(compressed_data, TEXTOID); + DecompressResult results_rev[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + memset(results_rev, 0xEF, sizeof(results_rev)); + + int rn = n - 1; + for (DecompressResult r = rev_iter->try_next(rev_iter); !r.is_done; + r = rev_iter->try_next(rev_iter)) + { + results_rev[rn--] = r; + } + if (bulk) { /* * Check that the arrow decompression result matches. */ decompress_generic_text_check_arrow(arrow, ERROR, results, n); + decompress_generic_text_check_arrow(arrow, ERROR, results_rev, n); return n; } + else + { + /* + * Check that the forward and reverse row-by-row decompression results match. + */ + decompress_generic_text_compare_results(results, results_rev, n); + } /* * For row-by-row decompression, check that the result is still the same