Add overflow protection to LZW buffer allocation

src/cgif_raw.c

@@ -332,7 +332,19 @@ static int LZW_GenerateStream(LZWResult* pResult, const uint32_t numPixel, const
   // where N = max dictionary resets = numPixel / (MAX_DICT_LEN - initDictLen - 2)
   entriesPerCycle = MAX_DICT_LEN - initDictLen - 2; // maximum added number of dictionary entries per cycle: -2 to account for start and end code
   maxResets = numPixel / entriesPerCycle;
-  pContext->pLZWData   = malloc(sizeof(uint16_t) * ((size_t)numPixel + 2 + maxResets));
+  /* Prevent overflow in addition */
+  if ((size_t)numPixel > SIZE_MAX - 2 - (size_t)maxResets) {
+    r = CGIF_EALLOC;
+    goto LZWGENERATE_Cleanup;
+  }
+  /* Safe addition */
+  size_t total_entries = (size_t)numPixel + 2 + (size_t)maxResets;
+  /* Prevent overflow in multiplication */
+  if (total_entries > SIZE_MAX / sizeof(uint16_t)) {
+    r = CGIF_EALLOC;
+    goto LZWGENERATE_Cleanup;
+  }
+  pContext->pLZWData = malloc(total_entries * sizeof(uint16_t));
   if(pContext->pLZWData == NULL) {
     r = CGIF_EALLOC;
     goto LZWGENERATE_Cleanup;

tests/lzw_alloc_overflow.c

@@ -0,0 +1,83 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "cgif_raw.h"
+
+/*
+ * Regression test for integer overflow in the LZW code-buffer allocation.
+ *
+ * LZW_GenerateStream() sizes pLZWData from:
+ *
+ *   (numPixel + 2 + maxResets) * sizeof(uint16_t)
+ *
+ * On a 32-bit target size_t is 32 bits wide, so for a numPixel close to
+ * UINT32_MAX (e.g. a 65535x65535 GIF is ~4.3 billion pixels) the additive
+ * expression wraps and malloc receives a drastically undersized request.
+ * LZW encoding then writes past the end of that buffer.
+ *
+ * The fix promotes the arithmetic to size_t and returns CGIF_EALLOC via the
+ * overflow guard before malloc is ever called.
+ *
+ * We reach the static function by including the translation unit directly and
+ * redirecting its malloc (same approach as ealloc_raw.c).
+ *
+ * Detecting the buffer without depending on malloc call order:
+ * pLZWData is the only allocation that scales with numPixel. Every other
+ * allocation is bounded by MAX_DICT_LEN and stays well under 64 KiB for the
+ * inputs used here, so we treat any request larger than that threshold as the
+ * LZW buffer. We record its size and return NULL so encoding never runs on a
+ * dummy image buffer.
+ *
+ *   main (unfixed), 32-bit:  request wraps to ~161 KiB (< numPixel bytes) -> FAIL
+ *   fixed, 32-bit:           guard returns CGIF_EALLOC, buffer never sized -> PASS
+ *   64-bit (either):         no wrap, request is > 8 GiB, we force NULL     -> PASS
+ */
+
+#define LZW_ALLOC_THRESHOLD 65536 /* every non-LZW allocation stays below this */
+
+static size_t lzwAllocSize; /* size of the LZW-buffer request, 0 if none seen */
+
+static void* test_malloc(size_t size) {
+  if(size > LZW_ALLOC_THRESHOLD) {
+    lzwAllocSize = size;
+    return NULL; /* abort before encoding a dummy image */
+  }
+  return malloc(size);
+}
+
+/* redirect malloc calls inside cgif_raw.c to our wrapper */
+#define malloc(s) test_malloc(s)
+#include "../src/cgif_raw.c"
+#undef malloc
+
+int main(void) {
+  LZWResult result;
+  /*
+   * numPixel large enough to overflow (numPixel + 2 + maxResets) in 32-bit
+   * size_t arithmetic. A 65535x65535 frame produces a value in this range.
+   */
+  const uint32_t numPixel    = 4294000000U;
+  const uint16_t initDictLen = 4; /* 1-color palette */
+  const uint8_t  initCodeLen = 3;
+
+  uint8_t imageData[1] = {0};
+
+  lzwAllocSize = 0;
+  memset(&result, 0, sizeof(result));
+
+  int r = LZW_GenerateStream(&result, numPixel, imageData, initDictLen, initCodeLen);
+
+  if(lzwAllocSize != 0 && lzwAllocSize < (size_t)numPixel) {
+    fprintf(stderr,
+            "FAIL: LZW buffer request wrapped (%zu bytes for %u pixels)\n",
+            lzwAllocSize, numPixel);
+    return 1;
+  }
+  if(r != CGIF_EALLOC) {
+    fprintf(stderr, "FAIL: expected CGIF_EALLOC, got %d\n", r);
+    return 1;
+  }
+  return 0;
+}

tests/meson.build

@@ -98,6 +98,14 @@ test_ealloc_rgb_exe = executable(
 )
 test('ealloc_rgb', test_ealloc_rgb_exe, priority : 0)
 
+# overflow guard test (compile source directly to intercept malloc)
+test_lzw_alloc_overflow_exe = executable(
+  'test_lzw_alloc_overflow',
+  'lzw_alloc_overflow.c',
+  include_directories : ['../inc/'],
+)
+test('lzw_alloc_overflow', test_lzw_alloc_overflow_exe, priority : 0)
+
 sha256sumc = find_program('scripts/sha256sum.py')
 # get the ordering right:
 # md5sum check on output GIFs should be run once all of the above tests are done.