Panic: Out of memory in enc.rs

[Yomihay-qut]

Description

Hello, I discovered a panic in pdf-rs while fuzzing.

The parser panics with "out-of-memory" when processing a specific input. It seems caused by unbounded decompression (Zip Bomb) and unchecked parameters leading to an allocation request that exceeds available memory.

Crash Location

• pdf/src/enc.rs:306
• Message: AddressSanitizer: out-of-memory

Fuzz Target

Here are the fuzz target and PoC that I used.

#![no_main]
use libfuzzer_sys::fuzz_target;
use pdf::enc::*;
use std::convert::TryInto;

fn get_i32(data: &[u8], offset: &mut usize) -> i32 {
    if *offset + 4 > data.len() {
        return 0;
    }
    let val = i32::from_le_bytes(data[*offset..*offset+4].try_into().unwrap());
    *offset += 4;
    val
}

fuzz_target!(|data: &[u8]| {
    if data.len() < 1 { return; }
    let mut offset = 0;
    
    let filter_type = data[offset] % 6; 
    offset += 1;

    let filter = match filter_type {
        0 => StreamFilter::ASCIIHexDecode,
        1 => StreamFilter::ASCII85Decode,
        2 | 3 => {
early_change
            let predictor = get_i32(data, &mut offset);
            let n_components = get_i32(data, &mut offset);
            let bits_per_component = get_i32(data, &mut offset);
            let columns = get_i32(data, &mut offset);
            let early_change = get_i32(data, &mut offset);
            
            let params = LZWFlateParams {
                predictor,
                n_components,
                bits_per_component,
                columns,
                early_change
            };
            
            if filter_type == 2 {
                StreamFilter::LZWDecode(params)
            } else {
                StreamFilter::FlateDecode(params)
            }
        },
        4 => StreamFilter::RunLengthDecode,
        5 => {
             let color_transform = Some(get_i32(data, &mut offset));
             StreamFilter::DCTDecode(DCTDecodeParams { color_transform })
        },
        _ => return
    };

    if offset > data.len() { return; }
    let payload = &data[offset..];
    
    let _ = decode(payload, &filter);
});

PoC.zip

Stack Trace

==101814==ERROR: AddressSanitizer: out of memory: allocator is trying to allocate 0x10160b5294 bytes
    #0 0x55e81641e169 in calloc /rustc/llvm/src/llvm-project/compiler-rt/lib/asan/asan_malloc_linux.cpp:74:3
    #1 0x55e8164a5b87 in alloc::alloc::alloc_zeroed /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/alloc.rs:178:9
    #2 0x55e8164a5b87 in <alloc::alloc::Global>::alloc_impl /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/alloc.rs:190:43
    #3 0x55e8164a5b87 in <alloc::alloc::Global as core::alloc::Allocator>::allocate_zeroed /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/alloc.rs:257:14
    #4 0x55e8164a5b87 in <alloc::raw_vec::RawVecInner>::try_allocate_in /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/raw_vec/mod.rs:470:40
    #5 0x55e8164a5b87 in <alloc::raw_vec::RawVecInner>::with_capacity_zeroed_in /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/raw_vec/mod.rs:443:15
    #6 0x55e8164a5b87 in <alloc::raw_vec::RawVec<u8>>::with_capacity_zeroed_in /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/raw_vec/mod.rs:208:20
    #7 0x55e8164a5b87 in <u8 as alloc::vec::spec_from_elem::SpecFromElem>::from_elem::<alloc::alloc::Global> /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/vec/spec_from_elem.rs:51:31
    #8 0x55e8164a5b87 in alloc::vec::from_elem::<u8> /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/alloc/src/vec/mod.rs:3510:5
    #9 0x55e8164a5b87 in pdf::enc::flate_decode /home/sun/pdf/pdf/src/enc.rs:306:24
    #10 0x55e8164a82c8 in pdf::enc::decode /home/sun/pdf/pdf/src/enc.rs:468:50
    #11 0x55e8164504eb in decode::_::__libfuzzer_sys_run /home/sun/pdf/pdf/fuzz/fuzz_targets/decode.rs:69:13
    #12 0x55e81645266a in rust_fuzzer_test_input /home/sun/.cargo/registry/src/rsproxy.cn-e3de039b2554c837/libfuzzer-sys-0.4.10/src/lib.rs:276:60
    #13 0x55e8165388e4 in libfuzzer_sys::test_input_wrap::{closure#0} /home/sun/.cargo/registry/src/rsproxy.cn-e3de039b2554c837/libfuzzer-sys-0.4.10/src/lib.rs:62:9
    #14 0x55e8165388e4 in std::panicking::catch_unwind::do_call::<libfuzzer_sys::test_input_wrap::{closure#0}, i32> /rustc/1aa9bab4ecbce4859eaad53000f78158ebe2be2c/library/std/src/panicking.rs:581:40
    #15 0x55e816539488 in __rust_try libfuzzer_sys.27fa543dd454df52-cgu.0