add malloc failure test for cgif_rgb API (dloebl#95)

* add malloc failure test for cgif_rgb API

* rgb: add missing alloc error checks

* make the rgb test cover more allocs

* address review feedback from other PRs

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* improve test

* simplify ealloc_rgb test

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: dloebl

src/cgif_rgb.c

@@ -126,40 +126,60 @@ static int32_t col_hash_collision_count(const uint8_t* rgb, const uint8_t* hashT
   return h;
 }
 
+/* free the color hash table*/
+static void free_col_hash_table(colHashTable* colhash){
+  free(colhash->frequ);
+  free(colhash->hashTable);
+  free(colhash->indexUsed);
+  free(colhash->pPalette);
+  free(colhash->colIdx);
+  free(colhash);
+}
+
 /* initialize hash table storing colors and their frequency */
 static colHashTable* init_col_hash_table(uint32_t tableSize){
   colHashTable* colhash = malloc(sizeof(colHashTable));
+  if(colhash == NULL) {
+    return NULL;
+  }
+  memset(colhash, 0, sizeof(colHashTable));
   colhash->tableSize = getNextPrimePower2(tableSize); // increase table size to next prime number
   colhash->frequ = malloc(sizeof(uint32_t) * colhash->tableSize);
   colhash->hashTable = malloc(3 * colhash->tableSize);
   colhash->indexUsed = malloc(colhash->tableSize);
   colhash->pPalette = malloc(3 * colhash->tableSize);
   colhash->colIdx = malloc(sizeof(uint32_t)*colhash->tableSize);
+  if(!colhash->frequ || !colhash->hashTable || !colhash->indexUsed || !colhash->pPalette || !colhash->colIdx) {
+    free_col_hash_table(colhash);
+    return NULL;
+  }
   colhash->cnt = 0;                                      // no colors initially
   memset(colhash->pPalette, 0, 3 * colhash->tableSize);  // unused part of color table is uninitialized otheriwse
   memset(colhash->indexUsed, 0, colhash->tableSize);     // initially no entry in hash-table is used
   return colhash;
 }
 
-/* free the color hash table*/
-static void free_col_hash_table(colHashTable* colhash){
-  free(colhash->frequ);
-  free(colhash->hashTable);
-  free(colhash->indexUsed);
-  free(colhash->pPalette);
-  free(colhash->colIdx);
-  free(colhash);
-}
-
 /* increase the size of the color hash table */
-static void resize_col_hash_table(colHashTable* colhash){
+static int resize_col_hash_table(colHashTable* colhash){
   uint32_t tableSizeNew;
   tableSizeNew  = getNextPrimePower2(colhash->tableSize); // increase table size to the next prime number above the next power of two
-  colhash->pPalette      = realloc(colhash->pPalette, 3 * tableSizeNew);
-  colhash->colIdx        = realloc(colhash->colIdx, sizeof(uint32_t) * tableSizeNew);
+  uint8_t* pPaletteNew   = realloc(colhash->pPalette, 3 * tableSizeNew);
+  uint32_t* colIdxNew    = realloc(colhash->colIdx, sizeof(uint32_t) * tableSizeNew);
   uint8_t* hashTable_new = malloc(3 * tableSizeNew);
   uint8_t* indexUsed_new = malloc(tableSizeNew);
-  uint32_t* frequ_new = malloc(sizeof(uint32_t) * tableSizeNew);
+  uint32_t* frequ_new    = malloc(sizeof(uint32_t) * tableSizeNew);
+  if(!pPaletteNew || !colIdxNew || !hashTable_new || !indexUsed_new || !frequ_new) {
+    // On allocation failure, apply any successful reallocs to colhash and free
+    // any newly allocated auxiliary buffers before returning an error.
+    if(pPaletteNew) colhash->pPalette = pPaletteNew;
+    if(colIdxNew)   colhash->colIdx   = colIdxNew;
+    free(hashTable_new);
+    free(indexUsed_new);
+    free(frequ_new);
+    return -1;
+  }
+  colhash->pPalette = pPaletteNew;
+  colhash->colIdx   = colIdxNew;
   memset(indexUsed_new, 0, tableSizeNew);
   colhash->cnt = 0;
   for(uint32_t j = 0; j < colhash->tableSize; ++j) { // TBD (no improvement when tested): easier to loop over pPalette and also leave pPalette in place?, if indexUsed is also unnecessary then
@@ -180,13 +200,17 @@ static void resize_col_hash_table(colHashTable* colhash){
   colhash->hashTable = hashTable_new; // pass pointer to new hash table
   colhash->indexUsed = indexUsed_new; // pass pointer to new hash table
   colhash->frequ = frequ_new; // pass pointer to new hash table
+  return 0;
 }
 
 /* take frequ indexed by hash(rgb) and return corresponding dense array */
 static uint32_t* hash_to_dense(colHashTable* colhash, cgif_chan_fmt fmtChan) {
   uint32_t* frequDense = malloc(sizeof(uint32_t) * colhash->cnt);
   uint32_t h;
   (void)fmtChan;
+  if(frequDense == NULL) {
+    return NULL;
+  }
   for(uint32_t i = 0; i < colhash->cnt; ++i) {
     h = col_hash(colhash->pPalette + 3 * i, colhash->hashTable, colhash->indexUsed, colhash->tableSize, 3);
     frequDense[i] = colhash->frequ[h];
@@ -243,6 +267,9 @@ static treeNode* new_tree_node(uint8_t* pPalette, uint32_t* frequ, uint16_t* num
   float var[3];
 
   treeNode* node = malloc(sizeof(treeNode));
+  if(node == NULL) {
+    return NULL;
+  }
   node->idxMin   = idxMin; // minimum color in pPalette belonging to the node
   node->idxMax   = idxMax; // maximum color in pPalette belonging to the node
   get_variance(pPalette, frequ, idxMin, idxMax, var, node->mean);
@@ -254,7 +281,7 @@ static treeNode* new_tree_node(uint8_t* pPalette, uint32_t* frequ, uint16_t* num
 }
 
 /* create the decision tree. (Similar to qsort with limited depth: pPalette, frequ get sorted) */
-static void crawl_decision_tree(treeNode* root, uint16_t* numLeaveNodes, uint8_t* pPalette, uint32_t* frequ, uint16_t colMax) {
+static int crawl_decision_tree(treeNode* root, uint16_t* numLeaveNodes, uint8_t* pPalette, uint32_t* frequ, uint16_t colMax) {
   uint32_t i, k, saveNum;
   uint16_t nodeIdx = 0;
   uint8_t saveBlk[3];
@@ -283,10 +310,14 @@ static void crawl_decision_tree(treeNode* root, uint16_t* numLeaveNodes, uint8_t
     (*numLeaveNodes)--;  // decrease counter since parent is removed as a leave node
     parent->child0 = new_tree_node(pPalette, frequ, numLeaveNodes, parent->idxMin, i - 1, parent->colIdx); // i-1 is last index of 1st block, one child takes color index from parent
     parent->child1 = new_tree_node(pPalette, frequ, numLeaveNodes, i, parent->idxMax, *numLeaveNodes);
+    if(parent->child0 == NULL || parent->child1 == NULL) {
+      return -1;
+    }
     nodeList[2*(*numLeaveNodes) - 3] = parent->child0; // add new child nodes to the list (total number of nodes is always 2*(*numLeaveNodes)-1)
     nodeList[2*(*numLeaveNodes) - 2] = parent->child1; // add new child nodes to the list (total number of nodes is always 2*(*numLeaveNodes)-1)
     }
   }
+  return 0;
 }
 
 /* fill 256 color table using the decision tree */
@@ -314,25 +345,34 @@ static uint8_t get_leave_node_index(const treeNode* root, const float* rgb) {
   }
 }
 
-/* color quantization with mean cut method (TBD? switch to median cut)
-   (works with dense palette, no hash table) */
-static treeNode* create_decision_tree(uint8_t* pPalette,  uint32_t* pFrequDense, uint8_t* pPalette256, uint32_t cnt, uint16_t colMax, uint8_t depthMax){
-  uint16_t numLeaveNodes = 0;
-  treeNode* root = new_tree_node(pPalette, pFrequDense, &numLeaveNodes, 0, cnt - 1, 0);
-  crawl_decision_tree(root, &numLeaveNodes, pPalette, pFrequDense, colMax);
-  get_palette_from_decision_tree(root, pPalette256); // fill the reduced color table
-  return root;
-}
-
 /* free memory allocated for the tree */
 static void free_decision_tree(treeNode* root){
+  if(root == NULL) {
+    return;
+  }
   if(root->isLeave == 0) { // if the node has children
     free_decision_tree(root->child0);
     free_decision_tree(root->child1);
   }
   free(root); // free root once free is called for children
 }
 
+/* color quantization with mean cut method (TBD? switch to median cut)
+   (works with dense palette, no hash table) */
+static treeNode* create_decision_tree(uint8_t* pPalette,  uint32_t* pFrequDense, uint8_t* pPalette256, uint32_t cnt, uint16_t colMax, uint8_t depthMax){
+  uint16_t numLeaveNodes = 0;
+  treeNode* root = new_tree_node(pPalette, pFrequDense, &numLeaveNodes, 0, cnt - 1, 0);
+  if(root == NULL) {
+    return NULL;
+  }
+  if(crawl_decision_tree(root, &numLeaveNodes, pPalette, pFrequDense, colMax) != 0) {
+    free_decision_tree(root);
+    return NULL;
+  }
+  get_palette_from_decision_tree(root, pPalette256); // fill the reduced color table
+  return root;
+}
+
 /* get image with max 256 color indices using Floyd-Steinberg dithering */
 static void get_quantized_dithered_image(uint8_t* pImageData, float* pImageDataRGBfloat, uint8_t* pPalette256, treeNode* root, uint32_t numPixel, uint32_t width, uint8_t dithering, uint8_t transIndex, cgif_chan_fmt fmtChan, uint8_t* pBef, cgif_chan_fmt befFmtChan, int hasAlpha) {
   // pImageData: image with (max 256) color indices (length: numPixel)
@@ -446,6 +486,9 @@ static colHashTable* get_color_histogram(const uint8_t* pImageDataRGB, uint32_t
   uint32_t cntCollision;                                    // count the number of collision
   uint32_t  tableSize = 262147;                             // initial size of the hash table
   colHashTable* colhash = init_col_hash_table(tableSize);   // initialize the hash table storing all the colors
+  if(colhash == NULL) {
+    return NULL;
+  }
   *pHasAlpha = 0;                                           // assume no alpha channel until it is found
   const uint8_t sizePixel = fmtChan;                        // number of bytes for one pixel (e.g. 3 for RGB, 4 for RGBa)
   for(uint32_t i = 0; i < numPixel; ++i) {
@@ -466,14 +509,17 @@ static colHashTable* get_color_histogram(const uint8_t* pImageDataRGB, uint32_t
     }
     // resize the hash table (if more than half-full)
     if((colhash->cnt > (colhash->tableSize >> 1) || cntCollision > MAX_COLLISIONS) && colhash->tableSize < MAX_TABLE_SIZE) {
-      resize_col_hash_table(colhash);
+      if(resize_col_hash_table(colhash) != 0) {
+        free_col_hash_table(colhash);
+        return NULL;
+      }
     }
   }
   return colhash;
 }
 
 /* quantize the image using the color histogram */
-static uint32_t quantize_and_dither(colHashTable* colhash, const uint8_t* pImageDataRGB, uint32_t numPixel, uint32_t width, cgif_chan_fmt fmtChan, uint8_t* pImageData, uint8_t* pPalette256, uint8_t depthMax, uint8_t dithering, int hasAlpha, uint8_t* pBef, cgif_chan_fmt befFmtChan) {
+static int quantize_and_dither(colHashTable* colhash, const uint8_t* pImageDataRGB, uint32_t numPixel, uint32_t width, cgif_chan_fmt fmtChan, uint8_t* pImageData, uint8_t* pPalette256, uint8_t depthMax, uint8_t dithering, int hasAlpha, uint8_t* pBef, cgif_chan_fmt befFmtChan) {
   // pImageDataRGB: image in RGB format
   // numPixel: number of pixels of input image
   // width: width of the image (needed for dithering)
@@ -488,9 +534,19 @@ static uint32_t quantize_and_dither(colHashTable* colhash, const uint8_t* pImage
   const uint16_t colMax = (1uL << depthMax) - 1; // maximum number of colors (-1 to leave space for transparency), disadvantage (TBD): quantization for static image with 256 colors and no alpha channel unnecessary
   if(colhash->cnt > colMax) { // color-quantization is needed
     uint32_t* pFrequDense = hash_to_dense(colhash, fmtChan);
+    if(pFrequDense == NULL) {
+      return -1;
+    }
     treeNode* root        = create_decision_tree(colhash->pPalette, pFrequDense, pPalette256, colhash->cnt, colMax, depthMax); // create decision tree (dynamic, splits along rgb-dimension with highest variance)
     free(pFrequDense);
+    if(root == NULL) {
+      return -1;
+    }
     float* pImageDataRGBfloat = malloc(fmtChan * numPixel * sizeof(float)); // TBD fmtChan + only when hasAlpha
+    if(pImageDataRGBfloat == NULL) {
+      free_decision_tree(root);
+      return -1;
+    }
     for(uint32_t i = 0; i < fmtChan * numPixel; ++i){
       pImageDataRGBfloat[i] = pImageDataRGB[i];
     }
@@ -521,6 +577,9 @@ CGIFrgb* cgif_rgb_newgif(const CGIFrgb_Config* pConfig) {
   CGIFrgb* pGIFrgb;
 
   pGIFrgb = malloc(sizeof(CGIFrgb));
+  if(pGIFrgb == NULL) {
+    return NULL;
+  }
   memset(pGIFrgb, 0, sizeof(CGIFrgb));
   idxConfig.pWriteFn  = pConfig->pWriteFn;
   idxConfig.pContext  = pConfig->pContext;
@@ -558,19 +617,40 @@ cgif_result cgif_rgb_addframe(CGIFrgb* pGIF, const CGIFrgb_FrameConfig* pConfig)
     return CGIF_ERROR;
   }
   pNewBef = malloc(pConfig->fmtChan * MULU16(imageWidth, imageHeight));
+  if(pNewBef == NULL) {
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
   memcpy(pNewBef, pConfig->pImageData, pConfig->fmtChan * MULU16(imageWidth, imageHeight));
   fConfig.pLocalPalette = aPalette;
   fConfig.pImageData    = malloc(pGIF->config.width * (uint32_t)pGIF->config.height);
+  if(fConfig.pImageData == NULL) {
+    free(pNewBef);
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
   fConfig.delay         = pConfig->delay;
   fConfig.attrFlags     = CGIF_FRAME_ATTR_USE_LOCAL_TABLE;
   if(pConfig->attrFlags & CGIF_RGB_FRAME_ATTR_INTERLACED) {
     fConfig.attrFlags |= CGIF_FRAME_ATTR_INTERLACED;
   }
 
   colHashTable* colhash = get_color_histogram(pConfig->pImageData, numPixel, pConfig->fmtChan, &hasAlpha);
+  if(colhash == NULL) {
+    free(pNewBef);
+    free(fConfig.pImageData);
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
   const uint8_t bDither = !(pConfig->attrFlags & CGIF_RGB_FRAME_ATTR_NO_DITHERING);
   const int sizeLCT = quantize_and_dither(colhash, pConfig->pImageData, numPixel, pGIF->config.width, pConfig->fmtChan, fConfig.pImageData, aPalette, 8, bDither, hasAlpha, pGIF->pBefImageData, pGIF->befFmtChan);
   free_col_hash_table(colhash);
+  if(sizeLCT < 0) {
+    free(pNewBef);
+    free(fConfig.pImageData);
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
 
   fConfig.numLocalPaletteEntries = sizeLCT;
   if(hasAlpha) {