tests: register lzw_alloc_overflow and make detection order-independent

Wire the LZW allocation overflow regression test into tests/meson.build so
it actually runs, and detect the buffer by size rather than by malloc call
index so the struct-of-arrays split in LZW_GenerateStream no longer moves the
target allocation. Fails on a 32-bit build without the guard, passes with it.

Signed-off-by: Uwez Khan <uwezkhan053@gmail.com>

Author: uwezkhan

tests/lzw_alloc_overflow.c

@@ -6,61 +6,43 @@
 #include "cgif_raw.h"
 
 /*
- * Regression test for integer overflow in LZW buffer allocation (PR #107).
+ * Regression test for integer overflow in the LZW code-buffer allocation.
  *
- * The original code computed the LZW data buffer size as:
+ * LZW_GenerateStream() sizes pLZWData from:
  *
  *   (numPixel + 2 + maxResets) * sizeof(uint16_t)
  *
- * using uint32_t arithmetic.  When numPixel approaches UINT32_MAX
- * (e.g. a 65535x65535 GIF = ~4.3 billion pixels), the additive expression
- * wraps, producing a drastically undersized malloc followed by a heap
- * buffer overflow during LZW encoding.
+ * On a 32-bit target size_t is 32 bits wide, so for a numPixel close to
+ * UINT32_MAX (e.g. a 65535x65535 GIF is ~4.3 billion pixels) the additive
+ * expression wraps and malloc receives a drastically undersized request.
+ * LZW encoding then writes past the end of that buffer.
  *
- * The fix promotes the arithmetic to size_t and adds explicit overflow
- * guards that return CGIF_EALLOC before calling malloc.
+ * The fix promotes the arithmetic to size_t and returns CGIF_EALLOC via the
+ * overflow guard before malloc is ever called.
  *
- * This test calls LZW_GenerateStream() directly (via source inclusion)
- * with a numPixel value large enough to trigger the overflow.  A malloc
- * wrapper intercepts the pLZWData allocation and verifies that its size
- * is not truncated.
+ * We reach the static function by including the translation unit directly and
+ * redirecting its malloc (same approach as ealloc_raw.c).
  *
- * Expected results:
- *   main (unfixed):  pLZWData malloc has a wrapped (tiny) size  => FAIL
- *   fixed:           overflow guard returns CGIF_EALLOC cleanly => PASS
+ * Detecting the buffer without depending on malloc call order:
+ * pLZWData is the only allocation that scales with numPixel. Every other
+ * allocation is bounded by MAX_DICT_LEN and stays well under 64 KiB for the
+ * inputs used here, so we treat any request larger than that threshold as the
+ * LZW buffer. We record its size and return NULL so encoding never runs on a
+ * dummy image buffer.
+ *
+ *   main (unfixed), 32-bit:  request wraps to ~161 KiB (< numPixel bytes) -> FAIL
+ *   fixed, 32-bit:           guard returns CGIF_EALLOC, buffer never sized -> PASS
+ *   64-bit (either):         no wrap, request is > 8 GiB, we force NULL     -> PASS
  */
 
-/* ---- malloc interception ---- */
+#define LZW_ALLOC_THRESHOLD 65536 /* every non-LZW allocation stays below this */
 
-static int   malloc_count;
-static int   overflow_detected;
-static size_t overflow_alloc_size;
+static size_t lzwAllocSize; /* size of the LZW-buffer request, 0 if none seen */
 
 static void* test_malloc(size_t size) {
-  ++malloc_count;
-  /*
-   * Inside LZW_GenerateStream the malloc calls are:
-   *   #1  pContext      (small)
-   *   #2  pTreeInit     (small)
-   *   #3  pTreeList     (small)
-   *   #4  pTreeMap      (small)
-   *   #5  pLZWData      <-- the one that overflows on main
-   *
-   * On the fixed branch the overflow guard fires between #4 and #5,
-   * so malloc_count never reaches 5.
-   *
-   * For #5 we check whether the requested size is suspiciously small.
-   * The correct size for numPixel ~ 4.3 billion is > 8 GB.
-   * A uint32-wrapped size would be < 1 MB.
-   * We always return NULL for #5 to prevent the function from trying
-   * to LZW-encode our dummy (1-byte) image buffer.
-   */
-  if (malloc_count == 5) {
-    overflow_alloc_size = size;
-    if (size < 100000000) { /* < 100 MB => uint32_t overflow occurred */
-      overflow_detected = 1;
-    }
-    return NULL; /* never allocate — image data is a dummy */
+  if(size > LZW_ALLOC_THRESHOLD) {
+    lzwAllocSize = size;
+    return NULL; /* abort before encoding a dummy image */
   }
   return malloc(size);
 }
@@ -73,43 +55,27 @@ static void* test_malloc(size_t size) {
 int main(void) {
   LZWResult result;
   /*
-   * numPixel that triggers uint32_t overflow in (numPixel + 2 + maxResets):
-   *
-   *   initDictLen    = 4   (for a 1-colour palette)
-   *   entriesPerCycle = MAX_DICT_LEN - 4 - 2 = 4090
-   *   maxResets       = 4294000000 / 4090 = 1049877
-   *   sum             = 4294000000 + 2 + 1049877 = 4295049879  (> UINT32_MAX)
-   *   wrapped uint32  = 82583
-   *   wrapped malloc  = 82583 * 2 = 165166 bytes   (~161 KB)
-   *
-   * On 32-bit (with fix):  size_t overflow guard fires   -> CGIF_EALLOC
-   * On 64-bit (with fix):  correct size_t addition (8.6 GB) -> malloc #5
-   *                         -> our wrapper returns NULL      -> CGIF_EALLOC
-   * On main  (no fix):     wrapped uint32 -> malloc #5 with 161 KB size
-   *                         -> overflow_detected = 1         -> FAIL
+   * numPixel large enough to overflow (numPixel + 2 + maxResets) in 32-bit
+   * size_t arithmetic. A 65535x65535 frame produces a value in this range.
    */
   const uint32_t numPixel    = 4294000000U;
-  const uint16_t initDictLen = 4;   /* 1 << (calcInitCodeLen(1) - 1) */
-  const uint8_t  initCodeLen = 3;   /* calcInitCodeLen(1) */
+  const uint16_t initDictLen = 4; /* 1-color palette */
+  const uint8_t  initCodeLen = 3;
 
-  /* Minimal image data — the overflow check fires before encoding. */
   uint8_t imageData[1] = {0};
 
-  malloc_count        = 0;
-  overflow_detected   = 0;
-  overflow_alloc_size = 0;
+  lzwAllocSize = 0;
   memset(&result, 0, sizeof(result));
 
   int r = LZW_GenerateStream(&result, numPixel, imageData, initDictLen, initCodeLen);
 
-  if (overflow_detected) {
+  if(lzwAllocSize != 0 && lzwAllocSize < (size_t)numPixel) {
     fprintf(stderr,
-            "FAIL: uint32 overflow in pLZWData allocation "
-            "(requested %zu bytes for %u pixels, expected > 8 GB)\n",
-            overflow_alloc_size, numPixel);
+            "FAIL: LZW buffer request wrapped (%zu bytes for %u pixels)\n",
+            lzwAllocSize, numPixel);
     return 1;
   }
-  if (r != CGIF_EALLOC) {
+  if(r != CGIF_EALLOC) {
     fprintf(stderr, "FAIL: expected CGIF_EALLOC, got %d\n", r);
     return 1;
   }

tests/meson.build

@@ -98,6 +98,14 @@ test_ealloc_rgb_exe = executable(
 )
 test('ealloc_rgb', test_ealloc_rgb_exe, priority : 0)
 
+# overflow guard test (compile source directly to intercept malloc)
+test_lzw_alloc_overflow_exe = executable(
+  'test_lzw_alloc_overflow',
+  'lzw_alloc_overflow.c',
+  include_directories : ['../inc/'],
+)
+test('lzw_alloc_overflow', test_lzw_alloc_overflow_exe, priority : 0)
+
 sha256sumc = find_program('scripts/sha256sum.py')
 # get the ordering right:
 # md5sum check on output GIFs should be run once all of the above tests are done.