From d8fe7a4f0456f02e5be9bce2ed7e8c4185674af0 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Thu, 12 Dec 2024 19:41:15 +0000 Subject: [PATCH 01/28] Added: bz3_memory_needed API. --- include/libbz3.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ src/libbz3.c | 25 ++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/include/libbz3.h b/include/libbz3.h index 4b31a38..df38afc 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -108,6 +108,60 @@ BZIP3_API int bz3_compress(uint32_t block_size, const uint8_t * in, uint8_t * ou */ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size); +/** + * @brief Calculate the total memory required for compression with the given block size. + * This includes all internal buffers and state structures. This calculates the amount of bytes + * that will be allocated by a call to `bz3_new()`. + * + * @details Memory allocation and usage patterns: + * + * bz3_new(): + * - Allocates all memory upfront: + * - Core state structure (sizeof(struct bz3_state)) + * - Swap buffer (bz3_bound(block_size) bytes) + * - SAIS array (BWT_BOUND(block_size) * sizeof(int32_t) bytes) + * - LZP lookup table ((1 << LZP_DICTIONARY) * sizeof(int32_t) bytes) + * - Compression state (sizeof(state)) + * - All memory remains allocated until bz3_free() + * + * Additional memory may be used depending on API used from here. + * + * # Low Level APIs + * + * 1. bz3_encode_block() / bz3_decode_block(): + * - Uses pre-allocated memory from bz3_new() + * - No additional memory allocation + * - Peak memory usage of physical RAM varies with compression stages: + * - LZP: Uses LZP lookup table + swap buffer + * - BWT: Uses SAIS array + swap buffer + * - Entropy coding: Uses compression state (cm_state) + swap buffer + * + * Using the higher level API, `bz3_compress`, expect an additional allocation + * of `bz3_bound(block_size)`. + * + * In the parallel version `bz3_encode_blocks`, each thread gets its own state, + * so memory usage is `n_threads * bz3_compress_memory_needed()`. + * + * # High Level APIs + * + * 1. bz3_compress(): + * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes) + * in addition to the memory amount returned by this method call. + * - Everything is freed after compression completes + * + * 2. bz3_decompress(): + * - Allocates additional temporary compression buffer (bz3_decode_bound(block_size) bytes) + * in addition to the memory amount returned by this method call. + * - Everything is freed after compression completes + * + * Memory remains constant during operation - no dynamic reallocation occurs + * during compression or decompression after initial setup. + * + * @param block_size The block size to be used for compression + * @return The total number of bytes required for compression, or 0 if block_size is invalid + */ +BZIP3_API size_t bz3_memory_needed(int32_t block_size); + /* ** LOW LEVEL APIs ** */ /** diff --git a/src/libbz3.c b/src/libbz3.c index 91bc272..d5e3989 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -915,3 +915,28 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, bz3_free(state); return BZ3_OK; } + +BZIP3_API size_t bz3_memory_needed(int32_t block_size) { + if (block_size < KiB(65) || block_size > MiB(511)) { + return 0; + } + + size_t total_size = 0; + + // This is based on bz3_new. + // Core state structure + total_size += sizeof(struct bz3_state); + + // cm_state + total_size += sizeof(state); + + // Swap buffer (needs to handle expanded size) (swap_buffer) + total_size += bz3_bound(block_size); + + // SAIS array + total_size += BWT_BOUND(block_size) * sizeof(int32_t); + + // LZP lookup table (lzp_lut) + total_size += (1 << LZP_DICTIONARY) * sizeof(int32_t); + return total_size; +} \ No newline at end of file From 4f851c4614faef0a1d8a8bf56dca96026544e10d Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Thu, 12 Dec 2024 20:12:54 +0000 Subject: [PATCH 02/28] Added: bz3_decode_block_bound --- include/libbz3.h | 12 ++++++++++++ src/libbz3.c | 10 +++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/libbz3.h b/include/libbz3.h index df38afc..fb934bc 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -199,6 +199,18 @@ BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], uint8_t * buffers[ BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], int32_t sizes[], int32_t orig_sizes[], int32_t n); +/** + * @brief Calculate the required output buffer size for decompressing a single block. + * + * When decompressing a block, additional space may be needed to handle internal + * headers from pre-filters like RLE and LZP. This function calculates the exact + * required output buffer size. + * + * @param orig_size The original (uncompressed) size of the block + * @return The required output buffer size for decompression + */ +BZIP3_API size_t bz3_decode_block_bound(size_t orig_size); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/libbz3.c b/src/libbz3.c index d5e3989..fbf0ab5 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -868,7 +868,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, struct bz3_state * state = bz3_new(block_size); if (!state) return BZ3_ERR_INIT; - u8 * compression_buf = malloc(bz3_bound(block_size)); + u8 * compression_buf = malloc(bz3_decode_block_bound(block_size)); if (!compression_buf) { bz3_free(state); return BZ3_ERR_INIT; @@ -939,4 +939,12 @@ BZIP3_API size_t bz3_memory_needed(int32_t block_size) { // LZP lookup table (lzp_lut) total_size += (1 << LZP_DICTIONARY) * sizeof(int32_t); return total_size; +} + +BZIP3_API size_t bz3_decode_block_bound(size_t orig_size) { + // Add 256 bytes to handle worst-case pre-filter headers: + // - RLE map (32 bytes for the bitmap) + // - LZP headers if present + // The 256-byte padding is confirmed by the author as sufficient + return orig_size + 256; } \ No newline at end of file From 94b6a3847d49e8445256a3dfdcfd605ae6df6cb0 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Thu, 12 Dec 2024 21:25:00 +0000 Subject: [PATCH 03/28] Renamed: bz3_decode_bound -> bz3_decode_block_bound in documentation --- include/libbz3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libbz3.h b/include/libbz3.h index fb934bc..1ed8f1c 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -150,7 +150,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, * - Everything is freed after compression completes * * 2. bz3_decompress(): - * - Allocates additional temporary compression buffer (bz3_decode_bound(block_size) bytes) + * - Allocates additional temporary compression buffer (bz3_decode_block_bound(block_size) bytes) * in addition to the memory amount returned by this method call. * - Everything is freed after compression completes * From 23952378ba320413f44139bd3275794896281347 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 02:40:09 +0000 Subject: [PATCH 04/28] Updated: libbz3.h to specify that it is a frame that is being compressed here. --- include/libbz3.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index 1ed8f1c..eb8facd 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -90,7 +90,7 @@ BZIP3_API size_t bz3_bound(size_t input_size); /* ** HIGH LEVEL APIs ** */ /** - * @brief Compress a block of data. This function does not support parallelism + * @brief Compress a frame. This function does not support parallelism * by itself, consider using the low level `bz3_encode_blocks()` function instead. * Using the low level API might provide better performance. * Returns a bzip3 error code; BZ3_OK when the operation is successful. @@ -100,7 +100,7 @@ BZIP3_API size_t bz3_bound(size_t input_size); BZIP3_API int bz3_compress(uint32_t block_size, const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size); /** - * @brief Decompress a block of data. This function does not support parallelism + * @brief Decompress a frame. This function does not support parallelism * by itself, consider using the low level `bz3_decode_blocks()` function instead. * Using the low level API might provide better performance. * Returns a bzip3 error code; BZ3_OK when the operation is successful. From ab885feae425eb40ec062324dfea101bb180be21 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:23:33 +0000 Subject: [PATCH 05/28] Reworked how the fuzzer script works. Can now generate its own input for easier debugging. --- examples/fuzz.c | 190 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 155 insertions(+), 35 deletions(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index 0485a39..841b560 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -1,4 +1,3 @@ - /* A tiny utility for fuzzing bzip3. * * Prerequisites: @@ -10,25 +9,25 @@ * * # Instructions: * - * 1. Build the Repository (per example in README.md) + * 1. Prepare fuzzer directories * - * This will get you a working binary of `bzip3` (in repo root). - * Then cd into this (examples) folder. + * mkdir -p afl_in && mkdir -p afl_out * - * 2. Prepare fuzzer directories + * 2. Build binary (to compress test data). * - * mkdir -p afl_in && mkdir -p afl_out + * afl-clang fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native * * 3. Make a fuzzer input file. * - * With `your_file` being an arbitrary input to test. + * With `your_file` being an arbitrary input to test, use this utility + * to generate a compressed test frame: * - * ../bzip3 -e your_file - * mv your_file.bz3 afl_in/ + * ./fuzz hl-api.c hl-api.c.bz3 8 + * mv hl-api.c.bz3 afl_in/ * - * 4. Build instrumented binary. + * 4. Build binary (for fuzzing). * - * afl-clang fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * afl-clang-fast fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native * * 5. Run the fuzzer. * @@ -46,41 +45,162 @@ #include #include #include +#include +#include + +#define KiB(x) ((x)*1024) + +// Required for AFL++ persistent mode +#ifdef __AFL_HAVE_MANUAL_CONTROL +#include +__AFL_FUZZ_INIT(); +#endif + +// Maximum allowed size to prevent excessive memory allocation +#define MAX_SIZE 0x10000000 // 256MB + +// Returns 0 on success, negative on input validation errors, positive on bzip3 errors +static int try_decompress(const uint8_t *input_buf, size_t input_len) { + if (input_len < 8) { // invalid, does not contain orig_size + return -1; + } + + size_t orig_size = *(const uint32_t *)input_buf; + uint8_t *outbuf = malloc(orig_size); + if (!outbuf) { + return -3; + } + + // We read orig_size from the input as we also want to fuzz it. + int bzerr = bz3_decompress( + input_buf + sizeof(uint32_t), + outbuf, + input_len - sizeof(uint32_t), + &orig_size + ); + + if (bzerr != BZ3_OK) { + printf("bz3_decompress() failed with error code %d\n", bzerr); + } else { + printf("OK, %d => %d\n", (int)input_len, (int)orig_size); + } + + free(outbuf); + return bzerr; +} + +static int compress_file(const char *infile, const char *outfile, uint32_t block_size) { + block_size = block_size <= KiB(65) ? KiB(65) : block_size; + + // Read the data into `inbuf` + FILE *fp_in = fopen(infile, "rb"); + if (!fp_in) { + perror("Failed to open input file"); + return 1; + } + + fseek(fp_in, 0, SEEK_END); + size_t insize = ftell(fp_in); + fseek(fp_in, 0, SEEK_SET); + + uint8_t *inbuf = malloc(insize); + if (!inbuf) { + fclose(fp_in); + return 1; + } + + fread(inbuf, 1, insize, fp_in); + fclose(fp_in); + + // Make buffer for output. + size_t outsize = bz3_bound(insize); + uint8_t *outbuf = malloc(outsize + sizeof(uint32_t)); + if (!outbuf) { + free(inbuf); + return 1; + } + + // Store original size at the start + // This is important, the `try_decompress` will read this field during fuzzing. + // And pass it as a parameter to `bz3_decompress`. + *(uint32_t *)outbuf = insize; + + int bzerr = bz3_compress(block_size, inbuf, outbuf + sizeof(uint32_t), insize, &outsize); + if (bzerr != BZ3_OK) { + printf("bz3_compress() failed with error code %d\n", bzerr); + free(inbuf); + free(outbuf); + return bzerr; + } + + FILE *fp_out = fopen(outfile, "wb"); + if (!fp_out) { + perror("Failed to open output file"); + free(inbuf); + free(outbuf); + return 1; + } + + fwrite(outbuf, 1, outsize + sizeof(uint32_t), fp_out); + fclose(fp_out); + + printf("Compressed %s (%zu bytes) to %s (%zu bytes)\n", + infile, insize, outfile, outsize + sizeof(uint32_t)); + + free(inbuf); + free(outbuf); + return 0; +} + +int main(int argc, char **argv) { +#ifdef __AFL_HAVE_MANUAL_CONTROL + __AFL_INIT(); + + while (__AFL_LOOP(1000)) { + try_decompress(__AFL_FUZZ_TESTCASE_BUF, __AFL_FUZZ_TESTCASE_LEN); + } +#else + if (argc == 4) { + // Compression mode: input_file output_file block_size + return compress_file(argv[1], argv[2], atoi(argv[3])); + } + + if (argc != 2) { + fprintf(stderr, "Usage:\n"); + fprintf(stderr, " Decompress: %s \n", argv[0]); + fprintf(stderr, " Compress: %s \n", argv[0]); + return 1; + } + + // Decompression mode + FILE *fp = fopen(argv[1], "rb"); + if (!fp) { + perror("Failed to open input file"); + return 1; + } -int main(int argc, char ** argv) { - // Read the entire input file to memory: - FILE * fp = fopen(argv[1], "rb"); fseek(fp, 0, SEEK_END); size_t size = ftell(fp); fseek(fp, 0, SEEK_SET); - volatile uint8_t * buffer = malloc(size); - fread(buffer, 1, size, fp); - fclose(fp); if (size < 64) { - // Too small. - free(buffer); + fclose(fp); return 0; } - // Decompress the file: - size_t orig_size = *(size_t *)buffer; - if (orig_size >= 0x10000000) { - // Sanity check: don't allocate more than 256MB. - free(buffer); - return 0; - } - uint8_t * outbuf = malloc(orig_size); - int bzerr = bz3_decompress(buffer + sizeof(size_t), outbuf, size - sizeof(size_t), &orig_size); - if (bzerr != BZ3_OK) { - printf("bz3_decompress() failed with error code %d", bzerr); - free(outbuf); - free(buffer); + uint8_t *buffer = malloc(size); + if (!buffer) { + fclose(fp); return 1; } - printf("OK, %d => %d", size, orig_size); - free(outbuf); + fread(buffer, 1, size, fp); + fclose(fp); + + int result = try_decompress(buffer, size); free(buffer); + return result > 0 ? result : 0; // Return bzip3 errors but treat validation errors as success +#endif + return 0; -} +} \ No newline at end of file From 06d0c77c5c3e189c1ceff1cacbd4862a1c82e9ab Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:26:55 +0000 Subject: [PATCH 06/28] Added: Multithreading to fuzz tutorial. --- examples/fuzz.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index 841b560..e9f073b 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -33,7 +33,18 @@ * * AFL_SKIP_CPUFREQ=1 afl-fuzz -i afl_in -o afl_out -- ./fuzz @@ * - * 6. Found a crash? + * 6. Wanna go faster? Multithread. + * + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -M fuzzer01 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer02 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer03 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer04 -- ./fuzz @@; exec bash" & + * + * etc. Replace `alacritty` with your terminal. + * + * And check progress with `afl-whatsup afl_out`. + * + * 7. Found a crash? * * If you find a crash, consider also doing the following: * From 3ee8e7bf546131a3467edb5c77a1210404497688 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:30:17 +0000 Subject: [PATCH 07/28] Added: 010editor template for fuzzer input. --- examples/fuzz.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/examples/fuzz.c b/examples/fuzz.c index e9f073b..a4d8690 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -53,6 +53,96 @@ * And run fuzz_asan on the crashing test case. Attach the test case /and/ the output of fuzz_asan to the bug report. */ + +/* + +This hex editor template can be used to help debug a breaking file. +Would provide for ImHex, but ImHex terminates if template is borked. + +//------------------------------------------------ +//--- 010 Editor v15.0.1 Binary Template +// +// File: bzip3.bt +// Authors: Sewer56 +// Version: 1.0.1 +// Purpose: Parse bzip3 fuzzer data +// Category: Archive +// File Mask: *.bz3 +// ID Bytes: 42 5A 33 76 31 // "BZ3v1" +//------------------------------------------------ + +// Colors for different sections +#define COLOR_HEADER 0xA0FFA0 // Frame header +#define COLOR_BLOCKHEAD 0xFFB0B0 // Block headers +#define COLOR_DATA 0xB0B0FF // Compressed data + +local uint32 currentBlockSize; // Store block size globally + +// Frame header structure +typedef struct { + char signature[5]; // "BZ3v1" + uint32 blockSize; // Maximum block size + uint32 block_count; +} FRAME_HEADER ; + +// Regular block header (for blocks >= 64 bytes) +typedef struct { + uint32 crc32; // CRC32 checksum of uncompressed data + uint32 bwtIndex; // Burrows-Wheeler transform index + uint8 model; // Compression model flags: + // bit 1 (0x02): LZP was used + // bit 2 (0x04): RLE was used + + // Optional size fields based on compression flags + if(model & 0x02) + uint32 lzpSize; // Size after LZP compression + if(model & 0x04) + uint32 rleSize; // Size after RLE compression +} BLOCK_HEADER ; + +// Small block header (for blocks < 64 bytes) +typedef struct { + uint32 crc32; // CRC32 checksum + uint32 literal; // Always 0xFFFFFFFF for small blocks + uint8 data[currentBlockSize - 8]; // Uncompressed data +} SMALL_BLOCK ; + +// Main block structure +typedef struct { + uint32 compressedSize; // Size of compressed block + uint32 origSize; // Original uncompressed size + + currentBlockSize = compressedSize; // Store for use in SMALL_BLOCK + + if(compressedSize < 64) { + SMALL_BLOCK content; + } else { + BLOCK_HEADER header; + uchar data[compressedSize - (Popcount(header.model) * 4 + 9)]; + } +} BLOCK ; + +// Helper function for bit counting (used for header size calculation) +int Popcount(byte b) { + local int count = 0; + while(b) { + count += b & 1; + b >>= 1; + } + return count; +} + +// Main parsing structure +uint32 frameCount; +FRAME_HEADER frameHeader; + +// Read blocks until end of file +while(!FEof()) { + BLOCK block; +} + +*/ + #include #include #include From df694f6e4b377e9251a91ec54c7ed981ca67fa30 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:39:24 +0000 Subject: [PATCH 08/28] Improved: Ignore the AFL Input/Outputs in .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ccccd80..788c967 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ examples/compress-file examples/decompress-file examples/fuzz +examples/afl_in +examples/afl_out bz3grep.1 From 56f5acf6a329c2334df878d021579532a606d730 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:41:23 +0000 Subject: [PATCH 09/28] Added a note saying `afl-whatsup` is not realtime. --- examples/fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index a4d8690..9b24df2 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -42,7 +42,7 @@ * * etc. Replace `alacritty` with your terminal. * - * And check progress with `afl-whatsup afl_out`. + * And check progress with `afl-whatsup afl_out` (updates periodically). * * 7. Found a crash? * From d29f8cc724d4d087bc3aec62c734a578bccccf38 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:45:05 +0000 Subject: [PATCH 10/28] Fixed: Variable name in embedded hex editor script. --- examples/fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index 9b24df2..f419fe3 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -133,7 +133,7 @@ int Popcount(byte b) { } // Main parsing structure -uint32 frameCount; +uint32 orig_size; FRAME_HEADER frameHeader; // Read blocks until end of file From 90f42efd3329c8933c17ce7eb6e96d98909bf78b Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 03:46:29 +0000 Subject: [PATCH 11/28] Added: Note on where to find crashing test case :3 --- examples/fuzz.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index f419fe3..617a1e9 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -50,7 +50,8 @@ * * clang fuzz.c ../src/libbz3.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address * - * And run fuzz_asan on the crashing test case. Attach the test case /and/ the output of fuzz_asan to the bug report. + * And run fuzz_asan on the crashing test case (you can find it in one of the `afl_out/crashes/` folders). + * Attach the test case /and/ the output of fuzz_asan to the bug report. */ From 52482a456fd1b06d724709d17ed2697fc52c2fd1 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 04:02:39 +0000 Subject: [PATCH 12/28] Added: Integration for bz3_decode_block_bound in bz3_decode_block --- src/libbz3.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libbz3.c b/src/libbz3.c index fbf0ab5..1445eff 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -620,7 +620,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s u32 crc32 = read_neutral_s32(buffer); s32 bwt_idx = read_neutral_s32(buffer + 4); - if (data_size > bz3_bound(state->block_size) || data_size < 0) { + if (data_size > bz3_decode_block_bound(state->block_size) || data_size < 0) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } @@ -651,13 +651,13 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s data_size -= p * 4 + 1; - if (((model & 2) && (lzp_size > bz3_bound(state->block_size) || lzp_size < 0)) || - ((model & 4) && (rle_size > bz3_bound(state->block_size) || rle_size < 0))) { + if (((model & 2) && (lzp_size > bz3_decode_block_bound(state->block_size) || lzp_size < 0)) || + ((model & 4) && (rle_size > bz3_decode_block_bound(state->block_size) || rle_size < 0))) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } - if (orig_size > bz3_bound(state->block_size) || orig_size < 0) { + if (orig_size > bz3_decode_block_bound(state->block_size) || orig_size < 0) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } @@ -698,7 +698,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s // Undo LZP if (model & 2) { - size_src = lzp_decompress(b1, b2, lzp_size, bz3_bound(state->block_size), state->lzp_lut); + size_src = lzp_decompress(b1, b2, lzp_size, bz3_decode_block_bound(state->block_size), state->lzp_lut); if (size_src == -1) { state->last_error = BZ3_ERR_CRC; return -1; From 55199ad4d7a2ae468e6a1e9d713383cfdaa253a0 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 06:35:14 +0000 Subject: [PATCH 13/28] Added: Guide for fuzzing with address sanitizer. --- examples/fuzz.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index 617a1e9..f9cec7b 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -48,10 +48,17 @@ * * If you find a crash, consider also doing the following: * - * clang fuzz.c ../src/libbz3.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address + * clang fuzz.c ../src/libbz3.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address * * And run fuzz_asan on the crashing test case (you can find it in one of the `afl_out/crashes/` folders). * Attach the test case /and/ the output of fuzz_asan to the bug report. + * + * If no error occurs, it could be that there was a memory corruption `between` the runs. + * In which case, you want to run AFL with address sanitizer. Use `export AFL_USE_ASAN=1` to enable + * addres sanitizer; then run AFL. + * + * export AFL_USE_ASAN=1 + * afl-clang-fast fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native */ From 645d11fbeba63da4abaa9f89cef21939a64dcd4b Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 08:47:05 +0000 Subject: [PATCH 14/28] Fixed: Wrong branch in binary template of fuzz.c --- examples/fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fuzz.c b/examples/fuzz.c index f9cec7b..afe8fa0 100644 --- a/examples/fuzz.c +++ b/examples/fuzz.c @@ -122,7 +122,7 @@ typedef struct { currentBlockSize = compressedSize; // Store for use in SMALL_BLOCK - if(compressedSize < 64) { + if(origSize < 64) { SMALL_BLOCK content; } else { BLOCK_HEADER header; From 24ea09691d8b71f2a55971f61d949e53a97813e7 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 09:58:58 +0000 Subject: [PATCH 15/28] Improved: Description in bz3_decode_block_bound --- src/libbz3.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libbz3.c b/src/libbz3.c index 1445eff..f967e7d 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -942,9 +942,11 @@ BZIP3_API size_t bz3_memory_needed(int32_t block_size) { } BZIP3_API size_t bz3_decode_block_bound(size_t orig_size) { - // Add 256 bytes to handle worst-case pre-filter headers: - // - RLE map (32 bytes for the bitmap) - // - LZP headers if present - // The 256-byte padding is confirmed by the author as sufficient + // Block may temporarily exceed the `orig_size` due to + // - RLE not being effective (encoded data pre commit 187b3228c73d4f35916ecb5950f18861ddc853ac) + // - LZP not being effective (encoded data pre commit 187b3228c73d4f35916ecb5950f18861ddc853ac) + // - Burrows Wheeler Transform (added bytes if RLE/LZP didn't save enough space) + // - Arithmetic Coding (on added bytes if prior steps failed) + // The 256-byte padding is claimed by the author as sufficient return orig_size + 256; } \ No newline at end of file From c3a78a46d01fe000c29bdb2a4212e872d9ecd2ea Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 22:19:50 +0000 Subject: [PATCH 16/28] Revert: data_size should not compare with bz3_decode_block_bound --- src/libbz3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libbz3.c b/src/libbz3.c index f967e7d..5c421cd 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -620,7 +620,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s u32 crc32 = read_neutral_s32(buffer); s32 bwt_idx = read_neutral_s32(buffer + 4); - if (data_size > bz3_decode_block_bound(state->block_size) || data_size < 0) { + if (data_size > bz3_bound(state->block_size) || data_size < 0) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } From 269133b5a7d0843936692fe7742786e23cbf9e91 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Fri, 13 Dec 2024 22:25:03 +0000 Subject: [PATCH 17/28] Removed: `bz3_decode_block_bound` API. --- include/libbz3.h | 14 +------------- src/libbz3.c | 20 +++++--------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index eb8facd..7da48cb 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -150,7 +150,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, * - Everything is freed after compression completes * * 2. bz3_decompress(): - * - Allocates additional temporary compression buffer (bz3_decode_block_bound(block_size) bytes) + * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes) * in addition to the memory amount returned by this method call. * - Everything is freed after compression completes * @@ -199,18 +199,6 @@ BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], uint8_t * buffers[ BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], int32_t sizes[], int32_t orig_sizes[], int32_t n); -/** - * @brief Calculate the required output buffer size for decompressing a single block. - * - * When decompressing a block, additional space may be needed to handle internal - * headers from pre-filters like RLE and LZP. This function calculates the exact - * required output buffer size. - * - * @param orig_size The original (uncompressed) size of the block - * @return The required output buffer size for decompression - */ -BZIP3_API size_t bz3_decode_block_bound(size_t orig_size); - #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/libbz3.c b/src/libbz3.c index 5c421cd..7057dc1 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -651,13 +651,13 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s data_size -= p * 4 + 1; - if (((model & 2) && (lzp_size > bz3_decode_block_bound(state->block_size) || lzp_size < 0)) || - ((model & 4) && (rle_size > bz3_decode_block_bound(state->block_size) || rle_size < 0))) { + if (((model & 2) && (lzp_size > bz3_bound(state->block_size) || lzp_size < 0)) || + ((model & 4) && (rle_size > bz3_bound(state->block_size) || rle_size < 0))) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } - if (orig_size > bz3_decode_block_bound(state->block_size) || orig_size < 0) { + if (orig_size > bz3_bound(state->block_size) || orig_size < 0) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } @@ -698,7 +698,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s // Undo LZP if (model & 2) { - size_src = lzp_decompress(b1, b2, lzp_size, bz3_decode_block_bound(state->block_size), state->lzp_lut); + size_src = lzp_decompress(b1, b2, lzp_size, bz3_bound(state->block_size), state->lzp_lut); if (size_src == -1) { state->last_error = BZ3_ERR_CRC; return -1; @@ -868,7 +868,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, struct bz3_state * state = bz3_new(block_size); if (!state) return BZ3_ERR_INIT; - u8 * compression_buf = malloc(bz3_decode_block_bound(block_size)); + u8 * compression_buf = malloc(bz3_bound(block_size)); if (!compression_buf) { bz3_free(state); return BZ3_ERR_INIT; @@ -940,13 +940,3 @@ BZIP3_API size_t bz3_memory_needed(int32_t block_size) { total_size += (1 << LZP_DICTIONARY) * sizeof(int32_t); return total_size; } - -BZIP3_API size_t bz3_decode_block_bound(size_t orig_size) { - // Block may temporarily exceed the `orig_size` due to - // - RLE not being effective (encoded data pre commit 187b3228c73d4f35916ecb5950f18861ddc853ac) - // - LZP not being effective (encoded data pre commit 187b3228c73d4f35916ecb5950f18861ddc853ac) - // - Burrows Wheeler Transform (added bytes if RLE/LZP didn't save enough space) - // - Arithmetic Coding (on added bytes if prior steps failed) - // The 256-byte padding is claimed by the author as sufficient - return orig_size + 256; -} \ No newline at end of file From a95e085ce92b91298cf7110fec3716171d4d26e2 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 00:16:07 +0000 Subject: [PATCH 18/28] Added: BZ3_ERR_DATA_SIZE_TOO_SMALL when bz3_decode_block is called with insufficient buffer. --- include/libbz3.h | 18 ++++++++++--- src/libbz3.c | 66 ++++++++++++++++++++++++++++++++++-------------- src/main.c | 20 +++++++++------ 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index 7da48cb..0f518ec 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -52,6 +52,7 @@ extern "C" { #define BZ3_ERR_TRUNCATED_DATA -5 #define BZ3_ERR_DATA_TOO_BIG -6 #define BZ3_ERR_INIT -7 +#define BZ3_ERR_ORIG_SIZE_TOO_SMALL -8 struct bz3_state; @@ -173,12 +174,21 @@ BZIP3_API int32_t bz3_encode_block(struct bz3_state * state, uint8_t * buffer, i /** * @brief Decode a single block. - * `buffer' must be able to hold at least `bz3_bound(orig_size)' bytes. The size must not exceed the block size - * associated with the state. + * + * `buffer' must be able to hold at least `bz3_bound(orig_size)' bytes + * in order to ensure decompression will succeed for all possible bzip3 blocks. + * + * In most (but not all) cases, `orig_size` should usually be sufficient. + * If it is not sufficient, you must allocate a buffer of size `bz3_bound(orig_size)` temporarily. + * + * If `buffer_size` is too small, `BZ3_ERR_ORIG_SIZE_TOO_SMALL` will be returned. + * The size must not exceed the block size associated with the state. + * + * @param buffer_size The size of the buffer at `buffer' * @param size The size of the compressed data in `buffer' * @param orig_size The original size of the data before compression. */ -BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, int32_t size, int32_t orig_size); +BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, size_t buffer_size, int32_t size, int32_t orig_size); /** * @brief Encode `n' blocks, all in parallel. @@ -196,7 +206,7 @@ BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], uint8_t * buffers[ * @brief Decode `n' blocks, all in parallel. * Same specifics as `bz3_encode_blocks', but doesn't overwrite `sizes'. */ -BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], int32_t sizes[], +BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], size_t buffer_sizes[], int32_t sizes[], int32_t orig_sizes[], int32_t n); #ifdef __cplusplus diff --git a/src/libbz3.c b/src/libbz3.c index 7057dc1..a7af992 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -489,6 +489,8 @@ BZIP3_API const char * bz3_strerror(struct bz3_state * state) { return "Truncated data"; case BZ3_ERR_DATA_TOO_BIG: return "Too much data"; + case BZ3_ERR_ORIG_SIZE_TOO_SMALL: + return "Size of buffer `buffer_size` passed to the block decoder (bz3_decode_block) is too small. See function docs for details."; default: return "Unknown error"; } @@ -615,7 +617,7 @@ BZIP3_API s32 bz3_encode_block(struct bz3_state * state, u8 * buffer, s32 data_s return data_size + overhead * 4 + 1; } -BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_size, s32 orig_size) { +BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 data_size, s32 orig_size) { // Read the header. u32 crc32 = read_neutral_s32(buffer); s32 bwt_idx = read_neutral_s32(buffer + 4); @@ -662,6 +664,36 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s return -1; } + // Size that undoing BWT+BCM should decompress into. + s32 size_before_bwt; + + if (model & 2) + size_before_bwt = lzp_size; + else if (model & 4) + size_before_bwt = rle_size; + else + size_before_bwt = orig_size; + + // Note(sewer): It's technically valid within the spec to create a bzip3 block + // where the size after LZP/RLE is larger than the original input. Some earlier encoders + // even (mistakenly?) were able to do this. + // + // SAFETY: Data passed to the BWT+BCM step can be one of the following: + // - original data + // - original data + LZP + // - original data + RLE + // - original data + RLE + LZP + // + // We must ensure `orig_size` is large enough to store the data at every step of the way + // when we walk backwards from undoing BWT+BCM. The size required may be stored in either `lzp_size`, + // `rle_size` OR `orig_size`. We therefore simply check all possible sizes. + size_t effective_lzp_size = lzp_size < 0 ? 0 : (size_t)lzp_size; // in case -1, we don't want implicit conversion + size_t effective_rle_size = rle_size < 0 ? 0 : (size_t)rle_size; + if ((effective_lzp_size > buffer_size) || (effective_rle_size > buffer_size)) { + state->last_error = BZ3_ERR_ORIG_SIZE_TOO_SMALL; + return -1; + } + // Decode the data. u8 *b1 = buffer, *b2 = state->swap_buffer; @@ -670,32 +702,25 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s state->cm_state->input_ptr = 0; state->cm_state->input_max = data_size; - s32 size_src; - - if (model & 2) - size_src = lzp_size; - else if (model & 4) - size_src = rle_size; - else - size_src = orig_size; - - decode_bytes(state->cm_state, b2, size_src); + decode_bytes(state->cm_state, b2, size_before_bwt); swap(b1, b2); - if (bwt_idx > size_src) { + if (bwt_idx > size_before_bwt) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } // Undo BWT memset(state->sais_array, 0, sizeof(s32) * BWT_BOUND(state->block_size)); - memset(b2, 0, size_src); - if (libsais_unbwt(b1, b2, state->sais_array, size_src, NULL, bwt_idx) < 0) { + memset(b2, 0, size_before_bwt); // buffer b2, swap b1 + if (libsais_unbwt(b1, b2, state->sais_array, size_before_bwt, NULL, bwt_idx) < 0) { state->last_error = BZ3_ERR_BWT; return -1; } swap(b1, b2); + s32 size_src = size_before_bwt; + // Undo LZP if (model & 2) { size_src = lzp_decompress(b1, b2, lzp_size, bz3_bound(state->block_size), state->lzp_lut); @@ -706,7 +731,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, s32 data_s swap(b1, b2); } - if (model & 4) { + if (model & 4) { int err = mrled(b1, b2, orig_size, size_src); if (err) { state->last_error = BZ3_ERR_CRC; @@ -748,6 +773,7 @@ typedef struct { typedef struct { struct bz3_state * state; u8 * buffer; + size_t buffer_size; s32 size; s32 orig_size; } decode_thread_msg; @@ -761,7 +787,7 @@ static void * bz3_init_encode_thread(void * _msg) { static void * bz3_init_decode_thread(void * _msg) { decode_thread_msg * msg = _msg; - bz3_decode_block(msg->state, msg->buffer, msg->size, msg->orig_size); + bz3_decode_block(msg->state, msg->buffer, msg->buffer_size, msg->size, msg->orig_size); pthread_exit(NULL); return NULL; // unreachable } @@ -779,12 +805,13 @@ BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], u8 * buffers[], s3 for (s32 i = 0; i < n; i++) sizes[i] = messages[i].size; } -BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], u8 * buffers[], s32 sizes[], s32 orig_sizes[], s32 n) { +BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], u8 * buffers[], size_t buffer_sizes[], s32 sizes[], s32 orig_sizes[], s32 n) { decode_thread_msg messages[n]; pthread_t threads[n]; for (s32 i = 0; i < n; i++) { messages[i].state = states[i]; messages[i].buffer = buffers[i]; + messages[i].buffer_size = buffer_sizes[i]; messages[i].size = sizes[i]; messages[i].orig_size = orig_sizes[i]; pthread_create(&threads[i], NULL, bz3_init_decode_thread, &messages[i]); @@ -868,7 +895,8 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, struct bz3_state * state = bz3_new(block_size); if (!state) return BZ3_ERR_INIT; - u8 * compression_buf = malloc(bz3_bound(block_size)); + size_t compression_buf_size = bz3_bound(block_size); + u8 * compression_buf = malloc(compression_buf_size); if (!compression_buf) { bz3_free(state); return BZ3_ERR_INIT; @@ -899,7 +927,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, return BZ3_ERR_DATA_TOO_BIG; } memcpy(compression_buf, in + 8, size); - bz3_decode_block(state, compression_buf, size, orig_size); + bz3_decode_block(state, compression_buf, compression_buf_size, size, orig_size); if (bz3_last_error(state) != BZ3_OK) { s8 last_error = state->last_error; bz3_free(state); diff --git a/src/main.c b/src/main.c index 9a5f7d5..86e2df4 100644 --- a/src/main.c +++ b/src/main.c @@ -229,7 +229,8 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size return 1; } - u8 * buffer = malloc(bz3_bound(block_size)); + size_t buffer_size = bz3_bound(block_size); + u8 * buffer = malloc(buffer_size); if (!buffer) { fprintf(stderr, "Failed to allocate memory.\n"); @@ -272,7 +273,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size } xread_noeof(buffer, 1, new_size, input_des); bytes_read += 8 + new_size; - if (bz3_decode_block(state, buffer, new_size, old_size) == -1) { + if (bz3_decode_block(state, buffer, buffer_size, new_size, old_size) == -1) { fprintf(stderr, "Failed to decode a block: %s\n", bz3_strerror(state)); return 1; } @@ -294,7 +295,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size } xread_noeof(buffer, 1, new_size, input_des); bytes_read += 8 + new_size; - if (bz3_decode_block(state, buffer, new_size, old_size) == -1) { + if (bz3_decode_block(state, buffer, buffer_size, new_size, old_size) == -1) { fprintf(stderr, "Writing invalid block: %s\n", bz3_strerror(state)); } xwrite(buffer, old_size, 1, output_des); @@ -315,7 +316,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size xread_noeof(buffer, 1, new_size, input_des); bytes_read += 8 + new_size; bytes_written += old_size; - if (bz3_decode_block(state, buffer, new_size, old_size) == -1) { + if (bz3_decode_block(state, buffer, buffer_size, new_size, old_size) == -1) { fprintf(stderr, "Failed to decode a block: %s\n", bz3_strerror(state)); return 1; } @@ -335,6 +336,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size struct bz3_state * states[workers]; u8 * buffers[workers]; s32 sizes[workers]; + size_t buffer_sizes[workers]; s32 old_sizes[workers]; for (s32 i = 0; i < workers; i++) { states[i] = bz3_new(block_size); @@ -342,7 +344,9 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size fprintf(stderr, "Failed to create a block encoder state.\n"); return 1; } - buffers[i] = malloc(block_size + block_size / 50 + 32); + size_t buffer_size = bz3_bound(block_size); + buffer_sizes[i] = buffer_size; + buffers[i] = malloc(buffer_size); if (!buffers[i]) { fprintf(stderr, "Failed to allocate memory.\n"); return 1; @@ -393,7 +397,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size xread_noeof(buffers[i], 1, sizes[i], input_des); bytes_read += 8 + sizes[i]; } - bz3_decode_blocks(states, buffers, sizes, old_sizes, i); + bz3_decode_blocks(states, buffers, buffer_sizes, sizes, old_sizes, i); for (s32 j = 0; j < i; j++) { if (bz3_last_error(states[j]) != BZ3_OK) { fprintf(stderr, "Failed to decode data: %s\n", bz3_strerror(states[j])); @@ -421,7 +425,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size xread_noeof(buffers[i], 1, sizes[i], input_des); bytes_read += 8 + sizes[i]; } - bz3_decode_blocks(states, buffers, sizes, old_sizes, i); + bz3_decode_blocks(states, buffers, buffer_sizes, sizes, old_sizes, i); for (s32 j = 0; j < i; j++) { if (bz3_last_error(states[j]) != BZ3_OK) { fprintf(stderr, "Writing invalid block: %s\n", bz3_strerror(states[j])); @@ -449,7 +453,7 @@ static int process(FILE * input_des, FILE * output_des, int mode, int block_size bytes_read += 8 + sizes[i]; bytes_written += old_sizes[i]; } - bz3_decode_blocks(states, buffers, sizes, old_sizes, i); + bz3_decode_blocks(states, buffers, buffer_sizes, sizes, old_sizes, i); for (s32 j = 0; j < i; j++) { if (bz3_last_error(states[j]) != BZ3_OK) { fprintf(stderr, "Failed to decode data: %s\n", bz3_strerror(states[j])); From 23fb30c867a764937976c4e87357f40224cdfc7f Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 01:50:00 +0000 Subject: [PATCH 19/28] Implemented: Additional bounds checks in `bz3_decode_block` and `bz3_orig_size_sufficient_for_decode` API --- include/libbz3.h | 27 ++++++++++++- src/libbz3.c | 100 +++++++++++++++++++++++++++++++++++++++-------- src/main.c | 2 +- 3 files changed, 109 insertions(+), 20 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index 0f518ec..40018a5 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -52,7 +52,7 @@ extern "C" { #define BZ3_ERR_TRUNCATED_DATA -5 #define BZ3_ERR_DATA_TOO_BIG -6 #define BZ3_ERR_INIT -7 -#define BZ3_ERR_ORIG_SIZE_TOO_SMALL -8 +#define BZ3_ERR_DATA_SIZE_TOO_SMALL -8 struct bz3_state; @@ -181,7 +181,7 @@ BZIP3_API int32_t bz3_encode_block(struct bz3_state * state, uint8_t * buffer, i * In most (but not all) cases, `orig_size` should usually be sufficient. * If it is not sufficient, you must allocate a buffer of size `bz3_bound(orig_size)` temporarily. * - * If `buffer_size` is too small, `BZ3_ERR_ORIG_SIZE_TOO_SMALL` will be returned. + * If `buffer_size` is too small, `BZ3_ERR_DATA_SIZE_TOO_SMALL` will be returned. * The size must not exceed the block size associated with the state. * * @param buffer_size The size of the buffer at `buffer' @@ -209,6 +209,29 @@ BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], uint8_t * buffers[ BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], size_t buffer_sizes[], int32_t sizes[], int32_t orig_sizes[], int32_t n); +/** + * @brief Check if using original file size as buffer size is sufficient for decompressing + * a block at `block` pointer. + * + * @param block Pointer to the compressed block data + * @param block_size Size of the block buffer in bytes (must be at least 13 bytes for header) + * @param orig_size Size of the original uncompressed data + * @return 1 if original size is sufficient, 0 if insufficient, -1 on header error (insufficient buffer size) + * + * @remarks + * + * This function is useful for external APIs using the low level block encoding API, + * `bz3_encode_block`. You would normally call this directly after `bz3_encode_block` + * on the block that has been output. + * + * The purpose of this function is to prevent encoding blocks that would require an additional + * malloc at decompress time. + * The goal is to prevent erroring with `BZ3_ERR_DATA_SIZE_TOO_SMALL`, thus + * in turn + */ +BZIP3_API int bz3_orig_size_sufficient_for_decode(const uint8_t * block, size_t block_size, int32_t orig_size); + + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/libbz3.c b/src/libbz3.c index a7af992..e222df6 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -87,6 +87,33 @@ static u32 lzp_upcast(const u8 * ptr) { return val; } +/** + * @brief Check if the buffer size is sufficient for decoding a bz3 block + * + * Data passed to the BWT+BCM step can be one of the following: + * - original data + * - original data + LZP + * - original data + RLE + * - original data + RLE + LZP + * + * We must ensure `buffer_size` is large enough to store the data at every step + * when walking backwards from undoing BWT+BCM. The required size may be stored in + * either `lzp_size`, `rle_size` OR `orig_size`. + * + * @param buffer_size Size of the output buffer + * @param lzp_size Size after LZP decompression (-1 if LZP not used) + * @param rle_size Size after RLE decompression (-1 if RLE not used) + * @return 1 if buffer size is sufficient, 0 otherwise + */ +static int bz3_check_buffer_size(size_t buffer_size, s32 lzp_size, s32 rle_size) { + // Handle -1 cases to avoid implicit conversion issues + size_t effective_lzp_size = lzp_size < 0 ? 0 : (size_t)lzp_size; + size_t effective_rle_size = rle_size < 0 ? 0 : (size_t)rle_size; + + // Check if buffer can hold intermediate results + return (effective_lzp_size <= buffer_size) && (effective_rle_size <= buffer_size); +} + static s32 lzp_encode_block(const u8 * RESTRICT in, const u8 * in_end, u8 * RESTRICT out, u8 * out_end, s32 * RESTRICT lut) { const u8 * ins = in; @@ -489,7 +516,7 @@ BZIP3_API const char * bz3_strerror(struct bz3_state * state) { return "Truncated data"; case BZ3_ERR_DATA_TOO_BIG: return "Too much data"; - case BZ3_ERR_ORIG_SIZE_TOO_SMALL: + case BZ3_ERR_DATA_SIZE_TOO_SMALL: return "Size of buffer `buffer_size` passed to the block decoder (bz3_decode_block) is too small. See function docs for details."; default: return "Unknown error"; @@ -618,6 +645,12 @@ BZIP3_API s32 bz3_encode_block(struct bz3_state * state, u8 * buffer, s32 data_s } BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 data_size, s32 orig_size) { + // Need minimum bytes for initial header + if (data_size < 9) { + state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; + return -1; + } + // Read the header. u32 crc32 = read_neutral_s32(buffer); s32 bwt_idx = read_neutral_s32(buffer + 4); @@ -633,6 +666,12 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf return -1; } + // Ensure there's enough space for the raw copied data. + if (data_size - 8 > buffer_size) { + state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; + return -1; + } + memmove(buffer, buffer + 8, data_size - 8); if (crc32sum(1, buffer, data_size - 8) != crc32) { @@ -644,11 +683,17 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf } s8 model = buffer[8]; - s32 lzp_size = -1, rle_size = -1, p = 0; + // Ensure we have sufficient bytes for the rle/lzp sizes. + size_t needed_header_size = 9 + ((model & 2) * 4) + ((model & 4) * 4); + if (buffer_size < needed_header_size) { + state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; + return -1; + } + + s32 lzp_size = -1, rle_size = -1, p = 0; if (model & 2) lzp_size = read_neutral_s32(buffer + 9 + 4 * p++); if (model & 4) rle_size = read_neutral_s32(buffer + 9 + 4 * p++); - p += 2; data_size -= p * 4 + 1; @@ -677,20 +722,8 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf // Note(sewer): It's technically valid within the spec to create a bzip3 block // where the size after LZP/RLE is larger than the original input. Some earlier encoders // even (mistakenly?) were able to do this. - // - // SAFETY: Data passed to the BWT+BCM step can be one of the following: - // - original data - // - original data + LZP - // - original data + RLE - // - original data + RLE + LZP - // - // We must ensure `orig_size` is large enough to store the data at every step of the way - // when we walk backwards from undoing BWT+BCM. The size required may be stored in either `lzp_size`, - // `rle_size` OR `orig_size`. We therefore simply check all possible sizes. - size_t effective_lzp_size = lzp_size < 0 ? 0 : (size_t)lzp_size; // in case -1, we don't want implicit conversion - size_t effective_rle_size = rle_size < 0 ? 0 : (size_t)rle_size; - if ((effective_lzp_size > buffer_size) || (effective_rle_size > buffer_size)) { - state->last_error = BZ3_ERR_ORIG_SIZE_TOO_SMALL; + if (!bz3_check_buffer_size(buffer_size, lzp_size, rle_size)) { + state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; return -1; } @@ -968,3 +1001,36 @@ BZIP3_API size_t bz3_memory_needed(int32_t block_size) { total_size += (1 << LZP_DICTIONARY) * sizeof(int32_t); return total_size; } + + +BZIP3_API int bz3_orig_size_sufficient_for_decode(const u8 * block, size_t block_size, s32 orig_size) { + // Need at least 9 bytes for the initial header (4 bytes BWT index + 4 bytes CRC + 1 byte model) + if (block_size < 9) { + return -1; + } + + s32 bwt_idx = read_neutral_s32(block + 4); + if (bwt_idx == -1) { + // Uncompressed literals. + // Original size always sufficient for uncompressed blocks + return 1; + } + + s8 model = block[8]; + s32 lzp_size = -1, rle_size = -1; + size_t header_size = 9; // Start after model byte + + // Ensure we have sufficient bytes for the rle/lzp sizes. + size_t needed_header_size = 9 + ((model & 2) * 4) + ((model & 4) * 4); + if (block_size < needed_header_size) { + return -1; + } + + // Need additional 4 bytes for each size field that might be present + if (model & 2) { + lzp_size = read_neutral_s32(block + header_size); + header_size += 4; + } + if (model & 4) rle_size = read_neutral_s32(block + header_size); + return bz3_check_buffer_size((size_t)orig_size, lzp_size, rle_size); +} \ No newline at end of file diff --git a/src/main.c b/src/main.c index 86e2df4..a449f8a 100644 --- a/src/main.c +++ b/src/main.c @@ -821,4 +821,4 @@ int main(int argc, char * argv[]) { } return r; -} +} \ No newline at end of file From 6e4f12def85b731dded259d5df0908a6fc16a515 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 02:17:47 +0000 Subject: [PATCH 20/28] Adjusted: Old fuzz script now named fuzz-decompress. Enforce includes from local files, not system. --- examples/{fuzz.c => fuzz-decompress.c} | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) rename examples/{fuzz.c => fuzz-decompress.c} (93%) diff --git a/examples/fuzz.c b/examples/fuzz-decompress.c similarity index 93% rename from examples/fuzz.c rename to examples/fuzz-decompress.c index afe8fa0..3d18a32 100644 --- a/examples/fuzz.c +++ b/examples/fuzz-decompress.c @@ -1,4 +1,4 @@ -/* A tiny utility for fuzzing bzip3. +/* A tiny utility for fuzzing bzip3 frame decompression. * * Prerequisites: * @@ -15,7 +15,7 @@ * * 2. Build binary (to compress test data). * - * afl-clang fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * afl-clang fuzz-decompress.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native * * 3. Make a fuzzer input file. * @@ -27,7 +27,7 @@ * * 4. Build binary (for fuzzing). * - * afl-clang-fast fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * afl-clang-fast fuzz-decompress.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native * * 5. Run the fuzzer. * @@ -48,7 +48,7 @@ * * If you find a crash, consider also doing the following: * - * clang fuzz.c ../src/libbz3.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address + * clang fuzz-decompress.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address * * And run fuzz_asan on the crashing test case (you can find it in one of the `afl_out/crashes/` folders). * Attach the test case /and/ the output of fuzz_asan to the bug report. @@ -58,25 +58,21 @@ * addres sanitizer; then run AFL. * * export AFL_USE_ASAN=1 - * afl-clang-fast fuzz.c -I../include ../src/libbz3.c -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * afl-clang-fast fuzz-decompress.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native */ /* - This hex editor template can be used to help debug a breaking file. Would provide for ImHex, but ImHex terminates if template is borked. //------------------------------------------------ //--- 010 Editor v15.0.1 Binary Template // -// File: bzip3.bt +// File: bzip3-fuzz-decompress.bt // Authors: Sewer56 -// Version: 1.0.1 +// Version: 1.0.0 // Purpose: Parse bzip3 fuzzer data -// Category: Archive -// File Mask: *.bz3 -// ID Bytes: 42 5A 33 76 31 // "BZ3v1" //------------------------------------------------ // Colors for different sections @@ -151,7 +147,8 @@ while(!FEof()) { */ -#include +#include "../include/libbz3.h" +#include "../src/libbz3.c" #include #include #include From d4971ecde080cc99ed92a3f6f2138ee0471bd1be Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 02:24:03 +0000 Subject: [PATCH 21/28] Fixed: Bad quote usage in bz3_decode_block --- include/libbz3.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index 40018a5..c36f4b6 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -184,8 +184,8 @@ BZIP3_API int32_t bz3_encode_block(struct bz3_state * state, uint8_t * buffer, i * If `buffer_size` is too small, `BZ3_ERR_DATA_SIZE_TOO_SMALL` will be returned. * The size must not exceed the block size associated with the state. * - * @param buffer_size The size of the buffer at `buffer' - * @param size The size of the compressed data in `buffer' + * @param buffer_size The size of the buffer at 'buffer' + * @param size The size of the compressed data in 'buffer' * @param orig_size The original size of the data before compression. */ BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, size_t buffer_size, int32_t size, int32_t orig_size); From a0e0077bbb19b2fa1d4312dbdb3662737ad8c830 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 02:33:19 +0000 Subject: [PATCH 22/28] Added: Standard fuzzer inputs. --- examples/standard_test_files/63_byte_file.bin | 1 + examples/standard_test_files/65_byte_file.bin | 1 + examples/standard_test_files/readme.txt | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 examples/standard_test_files/63_byte_file.bin create mode 100644 examples/standard_test_files/65_byte_file.bin create mode 100644 examples/standard_test_files/readme.txt diff --git a/examples/standard_test_files/63_byte_file.bin b/examples/standard_test_files/63_byte_file.bin new file mode 100644 index 0000000..5c80e5d --- /dev/null +++ b/examples/standard_test_files/63_byte_file.bin @@ -0,0 +1 @@ +  !"#$%&'()0123456789@ABCDEFGHIPQRSTUVWXY`abc \ No newline at end of file diff --git a/examples/standard_test_files/65_byte_file.bin b/examples/standard_test_files/65_byte_file.bin new file mode 100644 index 0000000..5fc7824 --- /dev/null +++ b/examples/standard_test_files/65_byte_file.bin @@ -0,0 +1 @@ +  !"#$%&'()0123456789@ABCDEFGHIPQRSTUVWXY`abcde \ No newline at end of file diff --git a/examples/standard_test_files/readme.txt b/examples/standard_test_files/readme.txt new file mode 100644 index 0000000..ae7d134 --- /dev/null +++ b/examples/standard_test_files/readme.txt @@ -0,0 +1,4 @@ +This is a standard set of files to use as inputs for fuzzer testing: + +- 65_byte_file.bin: 65 bytes, all unique +- 63_byte_file.bin: 63 bytes, all unique From 510c4ca47bf9222912a199d96fb76fa93bcbecfa Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 03:30:21 +0000 Subject: [PATCH 23/28] Fixed: fuzz failures in decode_block and added a new fuzzer script for block decodes --- examples/fuzz-decode-block.c | 332 +++++++++++++++++++++++++++++++++++ src/libbz3.c | 19 +- 2 files changed, 346 insertions(+), 5 deletions(-) create mode 100644 examples/fuzz-decode-block.c diff --git a/examples/fuzz-decode-block.c b/examples/fuzz-decode-block.c new file mode 100644 index 0000000..68ba434 --- /dev/null +++ b/examples/fuzz-decode-block.c @@ -0,0 +1,332 @@ +/* A tiny utility for fuzzing bzip3 block decompression. + * + * Prerequisites: + * + * - AFL https://github.com/AFLplusplus/AFLplusplus + * - clang (part of LLVM) + * + * On Arch this is `pacman -S afl++ clang` + * + * # Instructions: + * + * 1. Prepare fuzzer directories + * + * mkdir -p afl_in && mkdir -p afl_out + * + * 2. Build binary (to compress test data). + * + * afl-clang fuzz-decode-block.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * + * 3. Make a fuzzer input file. + * + * With `your_file` being an arbitrary input to test, use this utility + * to generate a compressed test block: + * + * ./fuzz standard_test_files/63_byte_file.bin 63_byte_file.bin.bz3b 8 + * ./fuzz standard_test_files/65_byte_file.bin 65_byte_file.bin.bz3b 8 + * mv 63_byte_file.bin.bz3b afl_in/ + * mv 65_byte_file.bin.bz3b afl_in/ + * + * For this test, it is recommended to make 2 files, one that's <64 bytes and one that's >64 bytes. + * + * 4. Build binary (for fuzzing). + * + * afl-clang-fast fuzz-decode-block.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * + * 5. Run the fuzzer. + * + * AFL_SKIP_CPUFREQ=1 afl-fuzz -i afl_in -o afl_out -- ./fuzz @@ + * + * 6. Wanna go faster? Multithread. + * + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -M fuzzer01 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer02 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer03 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer04 -- ./fuzz @@; exec bash" & + * + * etc. Replace `alacritty` with your terminal. + * + * And check progress with `afl-whatsup afl_out` (updates periodically). + * + * 7. Found a crash? + * + * If you find a crash, consider also doing the following: + * + * clang fuzz-decode-block.c -g3 -O3 -march=native -o fuzz_asan -I../include "-DVERSION=\"0.0.0\"" -fsanitize=undefined -fsanitize=address + * + * And run fuzz_asan on the crashing test case (you can find it in one of the `afl_out/crashes/` folders). + * Attach the test case /and/ the output of fuzz_asan to the bug report. + * + * If no error occurs, it could be that there was a memory corruption `between` the runs. + * In which case, you want to run AFL with address sanitizer. Use `export AFL_USE_ASAN=1` to enable + * addres sanitizer; then run AFL. + * + * export AFL_USE_ASAN=1 + * afl-clang-fast fuzz-decode-block.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + */ + +/* + +This hex editor template can be used to help debug a breaking file. +Would provide for ImHex, but ImHex terminates if template is borked. + + +//------------------------------------------------ +//--- 010 Editor v15.0.1 Binary Template +// +// File: bzip3block.bt +// Authors: Sewer56 +// Version: 1.0.0 +// Purpose: Parse bzip3 fuzzer block data +// Category: Archive +// File Mask: *.bz3b +//------------------------------------------------ + +// Colors for different sections +#define COLOR_HEADER 0xA0FFA0 // Block metadata +#define COLOR_BLOCKHEAD 0xFFB0B0 // Block headers +#define COLOR_DATA 0xB0B0FF // Compressed data + +local uint32 currentBlockSize; // Store block size globally + +// Block metadata structure +typedef struct { + uint32 orig_size; // Original uncompressed size + uint32 comp_size; // Compressed size + uint32 buffer_size; // Size of decompression buffer +} BLOCK_META ; + +// Regular block header (for blocks >= 64 bytes) +typedef struct { + uint32 crc32; // CRC32 checksum of uncompressed data + uint32 bwtIndex; // Burrows-Wheeler transform index + uint8 model; // Compression model flags: + // bit 1 (0x02): LZP was used + // bit 2 (0x04): RLE was used + + // Optional size fields based on compression flags + if(model & 0x02) + uint32 lzpSize; // Size after LZP compression + if(model & 0x04) + uint32 rleSize; // Size after RLE compression +} BLOCK_HEADER ; + +// Small block header (for blocks < 64 bytes) +typedef struct { + uint32 crc32; // CRC32 checksum + uint32 literal; // Always 0xFFFFFFFF for small blocks + uint8 data[currentBlockSize - 8]; // Uncompressed data +} SMALL_BLOCK ; + +// Block content structure +typedef struct { + currentBlockSize = meta.comp_size; + + if(meta.orig_size < 64) { + SMALL_BLOCK content; + } else { + BLOCK_HEADER header; + uchar data[meta.comp_size - (Popcount(header.model) * 4 + 9)]; + } +} BLOCK_CONTENT ; + +// Helper function for bit counting (used for header size calculation) +int Popcount(byte b) { + local int count = 0; + while(b) { + count += b & 1; + b >>= 1; + } + return count; +} + +// Main block structure +typedef struct { + BLOCK_META meta; + BLOCK_CONTENT content; +} BLOCK; + +// Main parsing structure +BLOCK block; +*/ + +#include "../include/libbz3.h" +#include "../src/libbz3.c" +#include +#include +#include +#include + +#define KiB(x) ((x)*1024) + +// Required for AFL++ persistent mode +#ifdef __AFL_HAVE_MANUAL_CONTROL +#include +__AFL_FUZZ_INIT(); +#endif + +size_t min_size_t(size_t a, size_t b) { + return (a < b) ? a : b; +} + +// Returns 0 on success, positive on bzip3 errors +static int try_decode_block(const uint8_t *input_buf, size_t input_len) { + // Read whatever metadata we can get + uint32_t orig_size = 0; + uint32_t comp_size = 0; + uint32_t buffer_size = 0; + + if (input_len >= 4) orig_size = *(const uint32_t *)input_buf; + if (input_len >= 8) comp_size = *(const uint32_t *)(input_buf + 4); + if (input_len >= 12) buffer_size = *(const uint32_t *)(input_buf + 8); + + // Initialize state with minimum block size + struct bz3_state *state = bz3_new(KiB(65)); + if (!state) return 0; // not under test + + // Allocate buffer with fuzzer-provided size + uint8_t *buffer = malloc(buffer_size); + if (!buffer) { + bz3_free(state); + return 0; // not under test + } + + // Copy whatever compressed data we can get + size_t data_len = input_len > 12 ? input_len - 12 : 0; + if (data_len > 0) { + memcpy(buffer, input_buf + 12, min_size_t(data_len, (size_t)buffer_size)); + } + + // Attempt decompression with potentially invalid parameters + int bzerr = bz3_decode_block(state, buffer, buffer_size, comp_size, orig_size); + // and pray we don't crash :p + + free(buffer); + bz3_free(state); + return bzerr; +} + +static int encode_block(const char *infile, const char *outfile, uint32_t block_size) { + block_size = block_size <= KiB(65) ? KiB(65) : block_size; + + // Read input file + FILE *fp_in = fopen(infile, "rb"); + if (!fp_in) { + perror("Failed to open input file"); + return 1; + } + + fseek(fp_in, 0, SEEK_END); + size_t insize = ftell(fp_in); + fseek(fp_in, 0, SEEK_SET); + + uint8_t *inbuf = malloc(insize); + if (!inbuf) { + fclose(fp_in); + return 1; + } + + fread(inbuf, 1, insize, fp_in); + fclose(fp_in); + + // Initialize compression state + struct bz3_state *state = bz3_new(block_size); + if (!state) { + free(inbuf); + return 1; + } + + // Make output buffer + size_t outsize = bz3_bound(insize); + uint8_t *outbuf = malloc(outsize + 12); // +12 for metadata + if (!outbuf) { + bz3_free(state); + free(inbuf); + return 1; + } + + // Store metadata + *(uint32_t *)outbuf = insize; // Original size + *(uint32_t *)(outbuf + 8) = outsize; // Buffer size needed for decompression + + // Compress the block + int32_t comp_size = bz3_encode_block(state, outbuf + 12, insize); + if (comp_size < 0) { + printf("bz3_encode_block() failed with error code %d\n", comp_size); + bz3_free(state); + free(inbuf); + free(outbuf); + return comp_size; + } + + // Store compressed size + *(uint32_t *)(outbuf + 4) = comp_size; + + FILE *fp_out = fopen(outfile, "wb"); + if (!fp_out) { + perror("Failed to open output file"); + bz3_free(state); + free(inbuf); + free(outbuf); + return 1; + } + + fwrite(outbuf, 1, comp_size + 12, fp_out); + fclose(fp_out); + + printf("Encoded block from %s (%zu bytes) to %s (%d bytes)\n", + infile, insize, outfile, comp_size + 12); + + bz3_free(state); + free(inbuf); + free(outbuf); + return 0; +} + +int main(int argc, char **argv) { +#ifdef __AFL_HAVE_MANUAL_CONTROL + __AFL_INIT(); + + while (__AFL_LOOP(1000)) { + try_decode_block(__AFL_FUZZ_TESTCASE_BUF, __AFL_FUZZ_TESTCASE_LEN); + } +#else + if (argc == 4) { + // Compression mode: input_file output_file block_size + return encode_block(argv[1], argv[2], atoi(argv[3])); + } + + if (argc != 2) { + fprintf(stderr, "Usage:\n"); + fprintf(stderr, " Decode: %s \n", argv[0]); + fprintf(stderr, " Encode: %s \n", argv[0]); + return 1; + } + + // Decode mode + FILE *fp = fopen(argv[1], "rb"); + if (!fp) { + perror("Failed to open input file"); + return 1; + } + + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + uint8_t *buffer = malloc(size); + if (!buffer) { + fclose(fp); + return 1; + } + + fread(buffer, 1, size, fp); + fclose(fp); + + int result = try_decode_block(buffer, size); + free(buffer); + return result > 0 ? result : 0; // Return bzip3 errors but treat validation errors as success +#endif + + return 0; +} \ No newline at end of file diff --git a/src/libbz3.c b/src/libbz3.c index e222df6..36f07da 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -105,13 +105,14 @@ static u32 lzp_upcast(const u8 * ptr) { * @param rle_size Size after RLE decompression (-1 if RLE not used) * @return 1 if buffer size is sufficient, 0 otherwise */ -static int bz3_check_buffer_size(size_t buffer_size, s32 lzp_size, s32 rle_size) { +static int bz3_check_buffer_size(size_t buffer_size, s32 lzp_size, s32 rle_size, s32 orig_size) { // Handle -1 cases to avoid implicit conversion issues size_t effective_lzp_size = lzp_size < 0 ? 0 : (size_t)lzp_size; size_t effective_rle_size = rle_size < 0 ? 0 : (size_t)rle_size; + size_t effective_orig_size = orig_size < 0 ? 0 : (size_t)orig_size; // Check if buffer can hold intermediate results - return (effective_lzp_size <= buffer_size) && (effective_rle_size <= buffer_size); + return (effective_lzp_size <= buffer_size) && (effective_rle_size <= buffer_size) && (effective_orig_size <= buffer_size); } static s32 lzp_encode_block(const u8 * RESTRICT in, const u8 * in_end, u8 * RESTRICT out, u8 * out_end, @@ -646,7 +647,7 @@ BZIP3_API s32 bz3_encode_block(struct bz3_state * state, u8 * buffer, s32 data_s BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 data_size, s32 orig_size) { // Need minimum bytes for initial header - if (data_size < 9) { + if (buffer_size < 9) { state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; return -1; } @@ -722,7 +723,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf // Note(sewer): It's technically valid within the spec to create a bzip3 block // where the size after LZP/RLE is larger than the original input. Some earlier encoders // even (mistakenly?) were able to do this. - if (!bz3_check_buffer_size(buffer_size, lzp_size, rle_size)) { + if (!bz3_check_buffer_size(buffer_size, lzp_size, rle_size, orig_size)) { state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; return -1; } @@ -761,10 +762,18 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf state->last_error = BZ3_ERR_CRC; return -1; } + // SAFETY(sewer): An attacker formed bzip3 data which decompresses as valid lzp. + // The headers above were set to ones that pass validation (size within bounds), but the + // data itself tries to escape buffer_size. Don't allow it to. + if (size_src > buffer_size) { + state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; + return -1; + } swap(b1, b2); } if (model & 4) { + // SAFETY: mrled is capped at orig_size, which is in bounds. int err = mrled(b1, b2, orig_size, size_src); if (err) { state->last_error = BZ3_ERR_CRC; @@ -1032,5 +1041,5 @@ BZIP3_API int bz3_orig_size_sufficient_for_decode(const u8 * block, size_t block header_size += 4; } if (model & 4) rle_size = read_neutral_s32(block + header_size); - return bz3_check_buffer_size((size_t)orig_size, lzp_size, rle_size); + return bz3_check_buffer_size((size_t)orig_size, lzp_size, rle_size, orig_size); } \ No newline at end of file From f6820239f3fa2660147a481518427485c5ecae4a Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 03:37:19 +0000 Subject: [PATCH 24/28] Added: Sanity check that the user provided a valid compressed data size to bz3_decode_block --- include/libbz3.h | 4 ++-- src/libbz3.c | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index c36f4b6..2876af5 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -185,10 +185,10 @@ BZIP3_API int32_t bz3_encode_block(struct bz3_state * state, uint8_t * buffer, i * The size must not exceed the block size associated with the state. * * @param buffer_size The size of the buffer at 'buffer' - * @param size The size of the compressed data in 'buffer' + * @param compressed_size The size of the compressed data in 'buffer' * @param orig_size The original size of the data before compression. */ -BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, size_t buffer_size, int32_t size, int32_t orig_size); +BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, size_t buffer_size, int32_t compressed_size, int32_t orig_size); /** * @brief Encode `n' blocks, all in parallel. diff --git a/src/libbz3.c b/src/libbz3.c index 36f07da..b57c726 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -645,9 +645,9 @@ BZIP3_API s32 bz3_encode_block(struct bz3_state * state, u8 * buffer, s32 data_s return data_size + overhead * 4 + 1; } -BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 data_size, s32 orig_size) { - // Need minimum bytes for initial header - if (buffer_size < 9) { +BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 compressed_size, s32 orig_size) { + // Need minimum bytes for initial header, and compressed_size needs to fit within claimed buffer size. + if (buffer_size < 9 || buffer_size < compressed_size) { state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; return -1; } @@ -656,31 +656,31 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf u32 crc32 = read_neutral_s32(buffer); s32 bwt_idx = read_neutral_s32(buffer + 4); - if (data_size > bz3_bound(state->block_size) || data_size < 0) { + if (compressed_size > bz3_bound(state->block_size) || compressed_size < 0) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } if (bwt_idx == -1) { - if (data_size - 8 > 64 || data_size < 8) { + if (compressed_size - 8 > 64 || compressed_size < 8) { state->last_error = BZ3_ERR_MALFORMED_HEADER; return -1; } // Ensure there's enough space for the raw copied data. - if (data_size - 8 > buffer_size) { + if (compressed_size - 8 > buffer_size) { state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL; return -1; } - memmove(buffer, buffer + 8, data_size - 8); + memmove(buffer, buffer + 8, compressed_size - 8); - if (crc32sum(1, buffer, data_size - 8) != crc32) { + if (crc32sum(1, buffer, compressed_size - 8) != crc32) { state->last_error = BZ3_ERR_CRC; return -1; } - return data_size - 8; + return compressed_size - 8; } s8 model = buffer[8]; @@ -697,7 +697,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf if (model & 4) rle_size = read_neutral_s32(buffer + 9 + 4 * p++); p += 2; - data_size -= p * 4 + 1; + compressed_size -= p * 4 + 1; if (((model & 2) && (lzp_size > bz3_bound(state->block_size) || lzp_size < 0)) || ((model & 4) && (rle_size > bz3_bound(state->block_size) || rle_size < 0))) { @@ -734,7 +734,7 @@ BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buf begin(state->cm_state); state->cm_state->in_queue = b1 + p * 4 + 1; state->cm_state->input_ptr = 0; - state->cm_state->input_max = data_size; + state->cm_state->input_max = compressed_size; decode_bytes(state->cm_state, b2, size_before_bwt); swap(b1, b2); From ffdc3fb7e13b6b67513d4a3d74893e6b208417b3 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 04:27:04 +0000 Subject: [PATCH 25/28] Changed: bz3_memory_needed -> bz3_min_memory_needed --- include/libbz3.h | 17 ++++++++++------- src/libbz3.c | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/libbz3.h b/include/libbz3.h index 2876af5..af447e7 100644 --- a/include/libbz3.h +++ b/include/libbz3.h @@ -110,7 +110,7 @@ BZIP3_API int bz3_compress(uint32_t block_size, const uint8_t * in, uint8_t * ou BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size); /** - * @brief Calculate the total memory required for compression with the given block size. + * @brief Calculate the minimal memory required for compression with the given block size. * This includes all internal buffers and state structures. This calculates the amount of bytes * that will be allocated by a call to `bz3_new()`. * @@ -131,7 +131,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, * * 1. bz3_encode_block() / bz3_decode_block(): * - Uses pre-allocated memory from bz3_new() - * - No additional memory allocation + * - No additional memory allocation except for libsais (usually ~16KiB) * - Peak memory usage of physical RAM varies with compression stages: * - LZP: Uses LZP lookup table + swap buffer * - BWT: Uses SAIS array + swap buffer @@ -147,21 +147,24 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, * * 1. bz3_compress(): * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes) - * in addition to the memory amount returned by this method call. + * in addition to the memory amount returned by this method call and libsais. * - Everything is freed after compression completes * * 2. bz3_decompress(): * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes) - * in addition to the memory amount returned by this method call. + * in addition to the memory amount returned by this method call and libsais. * - Everything is freed after compression completes * - * Memory remains constant during operation - no dynamic reallocation occurs - * during compression or decompression after initial setup. + * Memory remains constant during operation, with except of some small allocations from libsais during + * BWT stage. That is not accounted by this function, though it usually amounts to ~16KiB, negligible. + * The worst case of BWT is 2*block_size technically speaking. + * + * No dynamic (re)allocation occurs outside of that. * * @param block_size The block size to be used for compression * @return The total number of bytes required for compression, or 0 if block_size is invalid */ -BZIP3_API size_t bz3_memory_needed(int32_t block_size); +BZIP3_API size_t bz3_min_memory_needed(int32_t block_size); /* ** LOW LEVEL APIs ** */ diff --git a/src/libbz3.c b/src/libbz3.c index b57c726..81de2f2 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -986,7 +986,7 @@ BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, return BZ3_OK; } -BZIP3_API size_t bz3_memory_needed(int32_t block_size) { +BZIP3_API size_t bz3_min_memory_needed(int32_t block_size) { if (block_size < KiB(65) || block_size > MiB(511)) { return 0; } From 5174b4ea7078d920e22f7511c5d5e307794e6b9e Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 04:41:32 +0000 Subject: [PATCH 26/28] Added: Round Trip Fuzzer --- examples/fuzz-round-trip.c | 164 +++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 examples/fuzz-round-trip.c diff --git a/examples/fuzz-round-trip.c b/examples/fuzz-round-trip.c new file mode 100644 index 0000000..7517c5f --- /dev/null +++ b/examples/fuzz-round-trip.c @@ -0,0 +1,164 @@ +/* A tiny utility for fuzzing bzip3 round-trip compression/decompression. + * + * Prerequisites: + * + * - AFL https://github.com/AFLplusplus/AFLplusplus + * - clang (part of LLVM) + * + * On Arch this is `pacman -S afl++ clang` + * + * # Instructions: + * + * 1. Prepare fuzzer directories + * + * mkdir -p afl_in && mkdir -p afl_out + * + * 2. Insert a test file to afl_in/ + * + * cp ./standard_test_files/63_byte_file.bin afl_in/ + * + * 3. Build binary (for fuzzing) + * + * afl-clang-fast fuzz-round-trip.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + * + * 4. Run the fuzzer + * + * AFL_SKIP_CPUFREQ=1 afl-fuzz -i afl_in -o afl_out -- ./fuzz @@ + * + * 5. Need to go faster? Multithread. + * + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -M fuzzer01 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer02 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer03 -- ./fuzz @@; exec bash" & + * alacritty -e bash -c "afl-fuzz -i afl_in -o afl_out -S fuzzer04 -- ./fuzz @@; exec bash" & + * + * etc. Replace `alacritty` with your terminal. + * + * 6. For ASAN testing: + * + * export AFL_USE_ASAN=1 + * afl-clang-fast fuzz-round-trip.c -I../include -o fuzz -g3 "-DVERSION=\"0.0.0\"" -O3 -march=native + */ + +#include "../include/libbz3.h" +#include "../src/libbz3.c" +#include +#include +#include +#include + +#define KiB(x) ((x)*1024) +#define DEFAULT_BLOCK_SIZE KiB(65) + +// Required for AFL++ persistent mode +#ifdef __AFL_HAVE_MANUAL_CONTROL +#include +__AFL_FUZZ_INIT(); +#endif + +// Function to emulate a crash for diagnostic purposes +static void __attribute__((noreturn)) crash_with_message(const char* msg) { + fprintf(stderr, "Emulating crash: %s\n", msg); + // Use abort() to generate a crash that ASAN and other tools can catch + abort(); +} + +// Returns 0 on success, crashes on failure +static int try_round_trip(const uint8_t *input_buf, size_t input_len) { + if (input_len == 0) return 0; + + // Use the larger of DEFAULT_BLOCK_SIZE or input_len + size_t block_size = input_len > DEFAULT_BLOCK_SIZE ? input_len : DEFAULT_BLOCK_SIZE; + + struct bz3_state *state = bz3_new(block_size); + if (!state) { + return -1; // allocation failures not tested. + } + + // Allocate buffer for both compression and decompression + // Using block_size to ensure we have enough space for both operations + size_t comp_buf_len = bz3_bound(input_len); + uint8_t *comp_buf = malloc(comp_buf_len); + if (!comp_buf) { + bz3_free(state); + return -1; // allocation failures not tested. + } + + // Step 0: Move input to compress buffer + memmove(comp_buf, input_buf, input_len); + + // Step 1: Compress the input + int32_t comp_size = bz3_encode_block(state, comp_buf, input_len); + if (comp_size < 0) { + bz3_free(state); + free(comp_buf); + crash_with_message("Compression failed"); + } + + // Step 2: Decompress + int bzerr = bz3_decode_block(state, comp_buf, comp_buf_len, comp_size, input_len); + if (bzerr < 0 || bzerr != input_len) { + bz3_free(state); + free(comp_buf); + crash_with_message("Decompression failed"); + } + + // Step 3: Compare + if (memcmp(input_buf, comp_buf, input_len) != 0) { + bz3_free(state); + free(comp_buf); + crash_with_message("Round-trip data mismatch"); + } + + bz3_free(state); + free(comp_buf); + return 0; +} + +static int test_file(const char *filename) { + FILE *fp = fopen(filename, "rb"); + if (!fp) { + perror("Failed to open input file"); + return 1; + } + + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + uint8_t *buffer = malloc(size); + if (!buffer) { + fclose(fp); + crash_with_message("Failed to allocate input buffer"); + } + + if (fread(buffer, 1, size, fp) != size) { + fclose(fp); + free(buffer); + crash_with_message("Failed to read input file"); + } + fclose(fp); + + int result = try_round_trip(buffer, size); + free(buffer); + return result; +} + +int main(int argc, char **argv) { +#ifdef __AFL_HAVE_MANUAL_CONTROL + __AFL_INIT(); + + while (__AFL_LOOP(1000)) { + try_round_trip(__AFL_FUZZ_TESTCASE_BUF, __AFL_FUZZ_TESTCASE_LEN); + } +#else + if (argc != 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + return 1; + } + + return test_file(argv[1]); +#endif + + return 0; +} \ No newline at end of file From 66cde0b900ca31cef19c7f0540f292c78e4d564b Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 14 Dec 2024 04:57:16 +0000 Subject: [PATCH 27/28] Fixed: Possible out of bounds read in `lzp_decode_block`. Mark exit branches as unlikely. --- src/libbz3.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libbz3.c b/src/libbz3.c index 81de2f2..f08444b 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -18,12 +18,18 @@ */ #include "libbz3.h" - #include #include - #include "libsais.h" +#if defined(__GNUC__) || defined(__clang__) + #define LIKELY(x) __builtin_expect(!!(x), 1) + #define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else + #define LIKELY(x) (x) + #define UNLIKELY(x) (x) +#endif + /* CRC32 implementation. Since CRC32 generally takes less than 1% of the runtime on real-world data (e.g. the Silesia corpus), I decided against using hardware CRC32. This implementation is simple, fast, fool-proof and good enough to be used with bzip3. */ @@ -201,21 +207,23 @@ static s32 lzp_decode_block(const u8 * RESTRICT in, const u8 * in_end, s32 * RES while (in < in_end && out < out_end) { u32 idx = (ctx >> 15 ^ ctx ^ ctx >> 3) & ((s32)(1 << LZP_DICTIONARY) - 1); - s32 val = lut[idx]; + s32 val = lut[idx]; // SAFETY: guaranteed to be in-bounds by & mask. lut[idx] = (s32)(out - outs); if (*in == MATCH && val > 0) { in++; + // SAFETY: 'in' is advanced here, but it may have been at last index in the case of untrusted bad data. + if (UNLIKELY(in == in_end)) return -1; if (*in != 255) { s32 len = LZP_MIN_MATCH; while (1) { - if (in == in_end) return -1; + if (UNLIKELY(in == in_end)) return -1; len += *in; if (*in++ != 254) break; } const u8 * ref = outs + val; const u8 * oe = out + len; - if (oe > out_end) oe = out_end; + if (UNLIKELY(oe > out_end)) oe = out_end; while (out < oe) *out++ = *ref++; From 47e8322d53b71edf847a7200536d0e819d08202f Mon Sep 17 00:00:00 2001 From: Kamila Szewczyk <27734421+kspalaiologos@users.noreply.github.com> Date: Sat, 14 Dec 2024 08:59:51 +0100 Subject: [PATCH 28/28] Update comment --- src/libbz3.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libbz3.c b/src/libbz3.c index f08444b..554aa36 100644 --- a/src/libbz3.c +++ b/src/libbz3.c @@ -96,15 +96,15 @@ static u32 lzp_upcast(const u8 * ptr) { /** * @brief Check if the buffer size is sufficient for decoding a bz3 block * - * Data passed to the BWT+BCM step can be one of the following: + * Data passed to the last step can be one of the following: * - original data * - original data + LZP * - original data + RLE * - original data + RLE + LZP * * We must ensure `buffer_size` is large enough to store the data at every step - * when walking backwards from undoing BWT+BCM. The required size may be stored in - * either `lzp_size`, `rle_size` OR `orig_size`. + * when walking backwards. The required size may be stored in either `lzp_size`, + * `rle_size` OR `orig_size`. * * @param buffer_size Size of the output buffer * @param lzp_size Size after LZP decompression (-1 if LZP not used) @@ -1050,4 +1050,4 @@ BZIP3_API int bz3_orig_size_sufficient_for_decode(const u8 * block, size_t block } if (model & 4) rle_size = read_neutral_s32(block + header_size); return bz3_check_buffer_size((size_t)orig_size, lzp_size, rle_size, orig_size); -} \ No newline at end of file +}