[Bug] Potential integer overflow bug in `src/lib/openjp2/dwt.c`

[ipwning]

Summary

A signed integer overflow vulnerability exists in the DWT (Discrete Wavelet Transform) decoder when processing malformed JPEG2000 files with partial tile decoding enabled.

Affected Version

• OpenJPEG v2.5.4 (current master)

Bug detail

opj_dwt_decode_partial_tile() function has macros below:

// dwt.c:2614
OPJ_S_off(0, off) += (OPJ_SS__off(0, off) + OPJ_SS__off(0, off) + 2) >> 2;

// dwt.c:2692
OPJ_D_off(i, off) += (OPJ_S__off(i, off) + OPJ_S__off(i + 1, off)) >> 1;

When two OPJ_INT32 wavelet coefficients are added, the result can overflow if both values are large (positive or negative). Since signed integer overflow is undefined behavior in C, this can lead to unpredictable results.

Root cause

Malformed JPEG2000 files can contain abnormally large wavelet coefficient values. During DWT inverse transform, these values are added together without overflow checking, triggering signed integer overflow.

Reproduction

Build with UBSan:

CC=clang cmake -DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
  -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" \
  -DBUILD_SHARED_LIBS=OFF ..
make -j4

Trigger:

./bin/opj_decompress -i poc.j2k -o out.bmp -r 1 -d 100,100,200,200

Output:

This bug is not severe enough to allow access such as out-of-bounds, but when an invalid image is input, it causes undefined behavior instead of handling it correctly, resulting in the image becoming corrupted.

PoC file:
PoC.zip

Simply unzip the PoC.zip file and then run the above command to test.

Recommend

It seems advisable to check whether the result of (OPJ_SS__off(0, off) + OPJ_SS__off(0, off) + 2) and the result of OPJ_D_off(i, off) += (OPJ_S__off(i, off) + OPJ_S__off(i + 1, off)) cause integer overflow.

Appendix

I found this bug using AFL++ fuzzer.

Harness

#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include <unistd.h>

#include "openjpeg.h"

__AFL_FUZZ_INIT();

typedef struct {
    const uint8_t* pabyData;
    size_t         nCurPos;
    size_t         nLength;
} MemFile;

static void ErrorCallback(const char *msg, void *client_data)
{
    (void)msg;
    (void)client_data;
}

static void WarningCallback(const char *msg, void *client_data)
{
    (void)msg;
    (void)client_data;
}

static void InfoCallback(const char *msg, void *client_data)
{
    (void)msg;
    (void)client_data;
}

static OPJ_SIZE_T ReadCallback(void *pBuffer, OPJ_SIZE_T nBytes, void *pUserData)
{
    MemFile *memFile = (MemFile *)pUserData;
    if (memFile->nCurPos >= memFile->nLength) {
        return (OPJ_SIZE_T)-1;
    }
    if (memFile->nCurPos + nBytes >= memFile->nLength) {
        size_t nToRead = memFile->nLength - memFile->nCurPos;
        memcpy(pBuffer, memFile->pabyData + memFile->nCurPos, nToRead);
        memFile->nCurPos = memFile->nLength;
        return nToRead;
    }
    if (nBytes == 0) {
        return (OPJ_SIZE_T)-1;
    }
    memcpy(pBuffer, memFile->pabyData + memFile->nCurPos, nBytes);
    memFile->nCurPos += nBytes;
    return nBytes;
}

static OPJ_BOOL SeekCallback(OPJ_OFF_T nBytes, void *pUserData)
{
    MemFile *memFile = (MemFile *)pUserData;
    memFile->nCurPos = (size_t)nBytes;
    return OPJ_TRUE;
}

static OPJ_OFF_T SkipCallback(OPJ_OFF_T nBytes, void *pUserData)
{
    MemFile *memFile = (MemFile *)pUserData;
    memFile->nCurPos += (size_t)nBytes;
    return nBytes;
}

static int fuzz_one_input(const uint8_t *buf, size_t len)
{
    if (len < 2) {
        return 0;
    }

    uint8_t fuzz_byte = buf[len - 1];
    OPJ_UINT32 cp_reduce = (fuzz_byte & 0x03);
    OPJ_UINT32 cp_layer = ((fuzz_byte >> 2) & 0x0F);

    opj_codec_t *pCodec = opj_create_decompress(OPJ_CODEC_J2K);
    if (!pCodec) {
        return 0;
    }

    opj_set_info_handler(pCodec, InfoCallback, NULL);
    opj_set_warning_handler(pCodec, WarningCallback, NULL);
    opj_set_error_handler(pCodec, ErrorCallback, NULL);

    opj_dparameters_t parameters;
    opj_set_default_decoder_parameters(&parameters);

    parameters.cp_reduce = cp_reduce;
    parameters.cp_layer = cp_layer;

    if (!opj_setup_decoder(pCodec, &parameters)) {
        opj_destroy_codec(pCodec);
        return 0;
    }

    opj_stream_t *pStream = opj_stream_create(1024, OPJ_TRUE);
    if (!pStream) {
        opj_destroy_codec(pCodec);
        return 0;
    }

    MemFile memFile;
    memFile.pabyData = buf;
    memFile.nLength = len;
    memFile.nCurPos = 0;

    opj_stream_set_user_data_length(pStream, len);
    opj_stream_set_read_function(pStream, ReadCallback);
    opj_stream_set_seek_function(pStream, SeekCallback);
    opj_stream_set_skip_function(pStream, SkipCallback);
    opj_stream_set_user_data(pStream, &memFile, NULL);

    opj_image_t *psImage = NULL;
    if (!opj_read_header(pStream, pCodec, &psImage)) {
        opj_destroy_codec(pCodec);
        opj_stream_destroy(pStream);
        opj_image_destroy(psImage);
        return 0;
    }

    opj_codestream_info_v2_t *cstr_info = opj_get_cstr_info(pCodec);
    if (cstr_info) {
        opj_destroy_cstr_info(&cstr_info);
    }

    opj_codestream_index_t *cstr_index = opj_get_cstr_index(pCodec);
    if (cstr_index) {
        opj_destroy_cstr_index(&cstr_index);
    }

    OPJ_UINT32 width = psImage->x1 - psImage->x0;
    OPJ_UINT32 height = psImage->y1 - psImage->y0;

    OPJ_UINT32 max_dim = 2048;
    OPJ_UINT32 width_to_read = (width > max_dim) ? max_dim : width;
    OPJ_UINT32 height_to_read = (height > max_dim) ? max_dim : height;

    OPJ_UINT32 x_offset = (fuzz_byte & 0x40) ? (width_to_read / 4) : 0;
    OPJ_UINT32 y_offset = (fuzz_byte & 0x80) ? (height_to_read / 4) : 0;

    if (width_to_read > 0 && height_to_read > 0) {
        if (opj_set_decode_area(pCodec, psImage,
                               psImage->x0 + x_offset,
                               psImage->y0 + y_offset,
                               psImage->x0 + x_offset + (width_to_read - x_offset),
                               psImage->y0 + y_offset + (height_to_read - y_offset))) {
            opj_decode(pCodec, pStream, psImage);
        }
    }

    opj_end_decompress(pCodec, pStream);
    opj_stream_destroy(pStream);
    opj_destroy_codec(pCodec);
    opj_image_destroy(psImage);

    return 0;
}

int main(int argc, char **argv)
{
#ifdef __AFL_HAVE_MANUAL_CONTROL
    __AFL_INIT();
#endif

    unsigned char *buf = __AFL_FUZZ_TESTCASE_BUF;

    while (__AFL_LOOP(10000)) {
        size_t len = __AFL_FUZZ_TESTCASE_LEN;
        fuzz_one_input(buf, len);
    }

    return 0;
}