add malloc failure test for cgif API (#94)

* add malloc failure test for cgif API

* set aFrames[2] to NULL to prevent double-free on allocation failure

* address review feedback

* address review feedback

Author: dloebl

src/cgif.c

@@ -550,6 +550,7 @@ int cgif_addframe(CGIF* pGIF, CGIF_FrameConfig* pConfig) {
     // keep the flushed frame in memory, as we might need it to write the next one.
     pGIF->aFrames[0] = pGIF->aFrames[1];
     pGIF->aFrames[1] = pGIF->aFrames[2];
+    pGIF->aFrames[2] = NULL;
   }
   // create new Frame struct + make a deep copy of pConfig.
   pNewFrame = malloc(sizeof(CGIF_Frame));

tests/ealloc.c

@@ -0,0 +1,153 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "cgif.h"
+#include "cgif_raw.h"
+
+#define WIDTH  2
+#define HEIGHT 2
+
+/* malloc failure injection */
+static int fail_after;
+static int malloc_count;
+
+static void* cgif_test_malloc(size_t size) {
+  if(fail_after > 0) {
+    ++malloc_count;
+    if(malloc_count == fail_after) {
+      return NULL;
+    }
+  }
+  return malloc(size);
+}
+
+/* redirect malloc calls inside library sources to our wrapper */
+#define malloc(s) cgif_test_malloc(s)
+#include "../src/cgif_raw.c"
+/* avoid duplicate static function name */
+#define calcNextPower2Ex cgif_calcNextPower2Ex
+#include "../src/cgif.c"
+#undef calcNextPower2Ex
+#undef malloc
+
+/* no-op write callback */
+static int writeFn(void* pContext, const uint8_t* pData, const size_t numBytes) {
+  (void)pContext;
+  (void)pData;
+  (void)numBytes;
+  return 0;
+}
+
+int main(void) {
+  CGIF*            pGIF;
+  CGIF_Config      gConfig;
+  CGIF_FrameConfig fConfig;
+  cgif_result      r;
+  uint8_t aPalette[] = {
+    0x00, 0x00, 0x00, // black
+    0xFF, 0xFF, 0xFF, // white
+  };
+  uint8_t aImageData0[WIDTH * HEIGHT];
+  uint8_t aImageData1[WIDTH * HEIGHT];
+  memset(aImageData0, 0, sizeof(aImageData0));
+  memset(aImageData1, 1, sizeof(aImageData1));
+
+  for(fail_after = 1; ; ++fail_after) {
+    memset(&gConfig, 0, sizeof(gConfig));
+    gConfig.pWriteFn                = writeFn;
+    gConfig.width                   = WIDTH;
+    gConfig.height                  = HEIGHT;
+    gConfig.pGlobalPalette          = aPalette;
+    gConfig.numGlobalPaletteEntries = 2;
+    gConfig.attrFlags               = CGIF_ATTR_IS_ANIMATED;
+
+    malloc_count = 0;
+
+    pGIF = cgif_newgif(&gConfig);
+    if(malloc_count >= fail_after) {
+      if(pGIF != NULL) {
+        fprintf(stderr, "expected NULL from cgif_newgif (fail_after=%d)\n", fail_after);
+        return 1;
+      }
+      continue;
+    }
+    if(pGIF == NULL) {
+      fputs("unexpected NULL from cgif_newgif\n", stderr);
+      return 1;
+    }
+
+    // frame 1: local color table + interlaced
+    memset(&fConfig, 0, sizeof(fConfig));
+    fConfig.pImageData             = aImageData0;
+    fConfig.attrFlags              = CGIF_FRAME_ATTR_USE_LOCAL_TABLE | CGIF_FRAME_ATTR_INTERLACED;
+    fConfig.pLocalPalette          = aPalette;
+    fConfig.numLocalPaletteEntries = 2;
+    r = cgif_addframe(pGIF, &fConfig);
+    if(malloc_count >= fail_after) {
+      if(r != CGIF_EALLOC) {
+        fprintf(stderr, "expected CGIF_EALLOC from cgif_addframe (frame 1), got: %d\n", r);
+        return 1;
+      }
+      cgif_close(pGIF);
+      continue;
+    }
+    if(r != CGIF_OK) {
+      fprintf(stderr, "unexpected error from cgif_addframe (frame 1): %d\n", r);
+      return 1;
+    }
+
+    // frame 2: diff window optimization
+    memset(&fConfig, 0, sizeof(fConfig));
+    fConfig.pImageData = aImageData1;
+    fConfig.genFlags   = CGIF_FRAME_GEN_USE_DIFF_WINDOW;
+    r = cgif_addframe(pGIF, &fConfig);
+    if(malloc_count >= fail_after) {
+      if(r != CGIF_EALLOC) {
+        fprintf(stderr, "expected CGIF_EALLOC from cgif_addframe (frame 2), got: %d\n", r);
+        return 1;
+      }
+      cgif_close(pGIF);
+      continue;
+    }
+    if(r != CGIF_OK) {
+      fprintf(stderr, "unexpected error from cgif_addframe (frame 2): %d\n", r);
+      return 1;
+    }
+
+    // frame 3: transparency optimization (triggers queue flush + separate malloc path)
+    memset(&fConfig, 0, sizeof(fConfig));
+    fConfig.pImageData = aImageData0;
+    fConfig.genFlags   = CGIF_FRAME_GEN_USE_TRANSPARENCY;
+    r = cgif_addframe(pGIF, &fConfig);
+    if(malloc_count >= fail_after) {
+      if(r != CGIF_EALLOC) {
+        fprintf(stderr, "expected CGIF_EALLOC from cgif_addframe (frame 3), got: %d\n", r);
+        return 1;
+      }
+      cgif_close(pGIF);
+      continue;
+    }
+    if(r != CGIF_OK) {
+      fprintf(stderr, "unexpected error from cgif_addframe (frame 3): %d\n", r);
+      return 1;
+    }
+
+    r = cgif_close(pGIF);
+    if(malloc_count >= fail_after) {
+      if(r != CGIF_EALLOC) {
+        fprintf(stderr, "expected CGIF_EALLOC from cgif_close, got: %d (fail_after=%d)\n", r, fail_after);
+        return 1;
+      }
+      continue;
+    }
+    if(r != CGIF_OK) {
+      fprintf(stderr, "unexpected error from cgif_close: %d\n", r);
+      return 1;
+    }
+    break;
+  }
+
+  return 0;
+}

tests/ealloc_raw.c

@@ -91,6 +91,11 @@ int main(void) {
       }
       break;
     }
+    // our injected failure was hit during close (e.g. LZW encoding)
+    if(r != CGIF_EALLOC) {
+      fprintf(stderr, "expected CGIF_EALLOC from cgif_raw_close, got: %d (n=%d)\n", r, fail_after);
+      return 1;
+    }
   }
 
   return 0;

tests/meson.build

@@ -76,6 +76,13 @@ foreach t : tests_index + tests_rgb
 endforeach
 
 # malloc failure tests (compile source directly to intercept malloc)
+test_ealloc_exe = executable(
+  'test_ealloc',
+  'ealloc.c',
+  include_directories : ['../inc/'],
+)
+test('ealloc', test_ealloc_exe, priority : 0)
+
 test_ealloc_raw_exe = executable(
   'test_ealloc_raw',
   'ealloc_raw.c',