fix integer overflow in cgif_rgb_addframe byte-size calculation

The multiplication fmtChan * numPixel in quantize_and_dither() and
cgif_rgb_addframe() was computed in uint32_t arithmetic, which overflows
for large RGBA images (e.g. 32768x32769x4 > UINT32_MAX). This caused
undersized heap allocations followed by out-of-bounds writes.

Fix by promoting the multiplication to size_t with an overflow check.
Add regression test (rgb_overflow_trigger) that exercises the edge case.

Author: rootvector2

src/cgif_rgb.c

@@ -542,12 +542,17 @@ static int quantize_and_dither(colHashTable* colhash, const uint8_t* pImageDataR
     if(root == NULL) {
       return -1;
     }
-    float* pImageDataRGBfloat = malloc(fmtChan * numPixel * sizeof(float)); // TBD fmtChan + only when hasAlpha
+    size_t numValues = (size_t)fmtChan * numPixel;
+    if(numPixel != 0 && numValues / numPixel != fmtChan) {
+      free_decision_tree(root);
+      return -1;
+    }
+    float* pImageDataRGBfloat = malloc(numValues * sizeof(float)); // TBD fmtChan + only when hasAlpha
     if(pImageDataRGBfloat == NULL) {
       free_decision_tree(root);
       return -1;
     }
-    for(uint32_t i = 0; i < fmtChan * numPixel; ++i){
+    for(size_t i = 0; i < numValues; ++i){
       pImageDataRGBfloat[i] = pImageDataRGB[i];
     }
     uint8_t transIndex = colMax;
@@ -616,12 +621,17 @@ cgif_result cgif_rgb_addframe(CGIFrgb* pGIF, const CGIFrgb_FrameConfig* pConfig)
     pGIF->curResult = CGIF_ERROR;
     return CGIF_ERROR;
   }
-  pNewBef = malloc(pConfig->fmtChan * MULU16(imageWidth, imageHeight));
+  size_t totalBytes = (size_t)pConfig->fmtChan * numPixel;
+  if(numPixel != 0 && totalBytes / numPixel != pConfig->fmtChan) {
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
+  pNewBef = malloc(totalBytes);
   if(pNewBef == NULL) {
     pGIF->curResult = CGIF_EALLOC;
     return pGIF->curResult;
   }
-  memcpy(pNewBef, pConfig->pImageData, pConfig->fmtChan * MULU16(imageWidth, imageHeight));
+  memcpy(pNewBef, pConfig->pImageData, totalBytes);
   fConfig.pLocalPalette = aPalette;
   fConfig.pImageData    = malloc(pGIF->config.width * (uint32_t)pGIF->config.height);
   if(fConfig.pImageData == NULL) {

tests/meson.build

@@ -57,6 +57,7 @@ tests_rgb = [
   { 'name' : 'rgb_255colors',                      'seed_should_fail' : false},
   { 'name' : 'rgb_256colors',                      'seed_should_fail' : false},
   { 'name' : 'rgb_256digit',                       'seed_should_fail' : false},
+  { 'name' : 'rgb_overflow_trigger',               'seed_should_fail' : false},
   { 'name' : 'rgb_interlaced',                     'seed_should_fail' : false},
   { 'name' : 'rgb_single_color',                   'seed_should_fail' : false},
   { 'name' : 'rgb_no_dithering',                   'seed_should_fail' : false},

tests/rgb_overflow_trigger.c

@@ -0,0 +1,57 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "cgif.h"
+
+static int pWriteFn(void* pContext, const uint8_t* pData, const size_t numBytes) {
+  (void)pContext;
+  (void)pData;
+  (void)numBytes;
+  return 0;
+}
+
+/* regression test: fmtChan * numPixel overflows uint32_t for large dimensions with RGBA.
+   on 32-bit platforms size_t is 32 bits, so the overflow check fires and CGIF_EALLOC is returned.
+   on 64-bit platforms the multiplication fits in size_t but processing a 4 GB image
+   would time out on CI, so we skip. */
+int main(void) {
+  if(sizeof(size_t) > 4) {
+    return 0; // skip on 64-bit: no uint32_t overflow possible in size_t
+  }
+
+  const uint16_t width  = 32768;
+  const uint16_t height = 32769; // width * height * 4 > UINT32_MAX
+
+  CGIFrgb_Config gConfig = {0};
+  gConfig.pWriteFn = pWriteFn;
+  gConfig.width    = width;
+  gConfig.height   = height;
+
+  CGIFrgb* pGIF = cgif_rgb_newgif(&gConfig);
+  if(pGIF == NULL) {
+    return 2;
+  }
+
+  size_t bufSize = (size_t)width * height * 4;
+  uint8_t* buf = malloc(bufSize);
+  if(buf == NULL) {
+    cgif_rgb_close(pGIF);
+    return 0; // skip: not enough memory
+  }
+  memset(buf, 0, bufSize);
+
+  CGIFrgb_FrameConfig fConfig = {0};
+  fConfig.pImageData = buf;
+  fConfig.fmtChan    = CGIF_CHAN_FMT_RGBA;
+  fConfig.delay      = 0;
+
+  cgif_result r = cgif_rgb_addframe(pGIF, &fConfig);
+  cgif_rgb_close(pGIF);
+  free(buf);
+  if(r == CGIF_EALLOC) {
+    return 0; // overflow correctly detected
+  }
+  return 1;
+}