chore: stable

Author: nyurik

README.md

@@ -32,7 +32,7 @@ let mut encoded = Vec::new();
 codec.encode(&input, &mut encoded).unwrap();
 
 let mut decoded = Vec::new();
-codec.decode(&encoded, &mut decoded).unwrap();
+codec.decode(&encoded, &mut decoded, None).unwrap();
 
 assert_eq!(decoded, input);
 ```
@@ -103,14 +103,13 @@ Rust block codecs require block-aligned input. `CompositeCodec` chains a block c
 
 ### C++ (`cpp` feature)
 
-All C++ codecs implement `AnyLenCodec` for `&[u32]`.
-Block-oriented codecs (`CppFastPFor128`, `CppFastPFor256`, `CppSimdFastPFor128`, `CppSimdFastPFor256`) implement both `BlockCodec` and `AnyLenCodec`; the C++ library exposes them as composites. Use them directly for arbitrary-length input.
-`u64`-capable codecs implement `BlockCodec64` with `encode64` / `decode64`.
+All C++ codecs are composite (any-length) and implement `AnyLenCodec` only.
+`u64`-capable codecs (`CppFastPFor128`, `CppFastPFor256`, `CppVarInt`) also implement `BlockCodec64` with `encode64` / `decode64`.
 
 | Codec                       | Notes                                                                  |
 |-----------------------------|------------------------------------------------------------------------|
-| `CppFastPFor128`            | `FastPFor + VByte` composite, 128-element blocks. Also supports `u64`. |
-| `CppFastPFor256`            | `FastPFor + VByte` composite, 256-element blocks. Also supports `u64`. |
+| `CppFastPFor128`            | `FastPFor + VByte` composite, 128-element blocks. Also supports `u64`.  |
+| `CppFastPFor256`            | `FastPFor + VByte` composite, 256-element blocks. Also supports `u64`.  |
 | `CppSimdFastPFor128`        | SIMD-optimized 128-element variant                                     |
 | `CppSimdFastPFor256`        | SIMD-optimized 256-element variant                                     |
 | `CppBP32`                   | Binary packing, 32-bit blocks                                          |

benches/bench_utils.rs

@@ -12,7 +12,7 @@
 use core::ops::Range;
 
 #[allow(unused_imports)]
-use fastpfor::{AnyLenCodec, BlockCodec, slice_to_blocks};
+use fastpfor::{AnyLenCodec, AsU32, BlockCodec, slice_to_blocks};
 use rand::rngs::StdRng;
 use rand::{RngExt as _, SeedableRng};
 
@@ -138,6 +138,21 @@ pub fn decompress<C: BlockCodec + Default>(
     out.len()
 }
 
+/// Decompress with any-length codec `C`, using `expected_len` for validation/pre-allocation.
+#[allow(dead_code)] // used by smoke_cpp_vs_rust
+pub fn decompress_anylen<C: AnyLenCodec + Default>(
+    compressed: &[u32],
+    expected_len: usize,
+    out: &mut Vec<u32>,
+) -> usize {
+    let mut codec = C::default();
+    out.clear();
+    codec
+        .decode(compressed, out, Some(expected_len.as_u32()))
+        .unwrap();
+    out.len()
+}
+
 // ---------------------------------------------------------------------------
 // Pre-computed fixtures
 // ---------------------------------------------------------------------------

benches/fastpfor_benchmark.rs

@@ -2,9 +2,9 @@
 
 use std::hint::black_box;
 
-#[cfg(feature = "cpp")]
-use bytemuck::cast_slice;
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+#[cfg(feature = "cpp")]
+use fastpfor::{AnyLenCodec, AsU32};
 use fastpfor::{BlockCodec as _, FastPForBlock128, FastPForBlock256, slice_to_blocks};
 
 #[path = "bench_utils.rs"]
@@ -186,8 +186,8 @@ fn benchmark_compression_ratio(c: &mut Criterion) {
     group.finish();
 }
 
-/// Compare encoding and decoding speed of the C++ `FastPForBlock128` codec against
-/// the pure-Rust `FastPForBlock128` codec.
+/// Compare encoding and decoding speed of the C++ `CppFastPFor128` (`AnyLenCodec`) against
+/// the pure-Rust `FastPForBlock128` (`BlockCodec`). Same wire format for block-aligned data.
 #[cfg(feature = "cpp")]
 fn benchmark_cpp_vs_rust(c: &mut Criterion) {
     let mut group = c.benchmark_group("cpp_vs_rust/encode");
@@ -199,11 +199,10 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
             &fix.data,
             |b, data| {
                 let mut codec = CppFastPFor128::default();
-                let blocks: &[[u32; 128]] = cast_slice(data);
                 let mut out = Vec::new();
                 b.iter(|| {
                     out.clear();
-                    codec.encode_blocks(black_box(blocks), &mut out).unwrap();
+                    codec.encode(black_box(data), &mut out).unwrap();
                     black_box(out.len())
                 });
             },
@@ -213,7 +212,7 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
             &fix.data,
             |b, data| {
                 let mut codec = FastPForBlock128::default();
-                let blocks: &[[u32; 128]] = cast_slice(data);
+                let (blocks, _) = slice_to_blocks::<FastPForBlock128>(data);
                 let mut out = Vec::new();
                 b.iter(|| {
                     out.clear();
@@ -228,6 +227,7 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
     let mut group = c.benchmark_group("cpp_vs_rust/decode");
     for (bc, fix) in compress_fixtures::<FastPForBlock128>(BLOCK_COUNTS) {
         let n_elem = fix.n_blocks * FastPForBlock128::values_per_block();
+        let expected_len = n_elem.as_u32();
         group.throughput(Throughput::Elements(n_elem as u64));
         group.bench_with_input(
             BenchmarkId::new(format!("cpp/{}", fix.name), bc),
@@ -238,7 +238,7 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
                 b.iter(|| {
                     out.clear();
                     codec
-                        .decode_blocks(black_box(compressed), fix.n_blocks, &mut out)
+                        .decode(black_box(compressed), &mut out, Some(expected_len))
                         .unwrap();
                     black_box(out.len())
                 });

fuzz/fuzz_targets/common.rs

@@ -49,12 +49,7 @@ macro_rules! codec_list {
 }
 
 /// Rust codecs. Block codecs are wrapped in `CompositeCodec<_, VariableByte>`.
-pub static RUST: &[CodecEntry] = codec_list!(
-    FastPFor256,
-    FastPFor128,
-    VariableByte,
-    JustCopy,
-);
+pub static RUST: &[CodecEntry] = codec_list!(FastPFor256, FastPFor128, VariableByte, JustCopy,);
 
 /// C++ codecs (any-length; block codecs are already composites in the C++ library).
 pub static CPP: &[CodecEntry] = codec_list!(

fuzz/fuzz_targets/cpp_roundtrip.rs

@@ -21,7 +21,7 @@ fuzz_target!(|data: FuzzInput<AnyLenSelector>| {
 
     let mut decompressed = Vec::new();
     codec
-        .decode(&compressed, &mut decompressed)
+        .decode(&compressed, &mut decompressed, None)
         .expect("C++ decompression failed");
 
     assert_eq!(

fuzz/fuzz_targets/decode_arbitrary.rs

@@ -57,5 +57,5 @@ fuzz_target!(|data: FuzzInput| {
 
     // The decoder must either succeed or return an error — a panic is a bug.
     let mut output = Vec::new();
-    let _ = codec.decode(&compressed, &mut output);
+    let _ = codec.decode(&compressed, &mut output, None);
 });

fuzz/fuzz_targets/decode_oracle.rs

@@ -41,7 +41,7 @@ fuzz_target!(|data: FuzzInput<CompatSelector>| {
     }
     let mut rust_decompressed = Vec::new();
     rust_codec
-        .decode(&rust_compressed, &mut rust_decompressed)
+        .decode(&rust_compressed, &mut rust_decompressed, None)
         .expect("Rust decompress of self-compressed data must not fail");
 
     // C++ roundtrip (independent oracle)
@@ -51,7 +51,7 @@ fuzz_target!(|data: FuzzInput<CompatSelector>| {
         .expect("C++ compression failed");
     let mut cpp_decompressed = Vec::new();
     cpp_codec
-        .decode(&cpp_compressed, &mut cpp_decompressed)
+        .decode(&cpp_compressed, &mut cpp_decompressed, None)
         .expect("C++ decompression failed");
 
     assert_eq!(

fuzz/fuzz_targets/encode_oracle.rs

@@ -25,7 +25,7 @@ fuzz_target!(|data: FuzzInput<AnyLenSelector>| {
 
     let mut decompressed = Vec::new();
     codec
-        .decode(&compressed, &mut decompressed)
+        .decode(&compressed, &mut decompressed, None)
         .expect("Rust decompress of self-compressed data must not fail");
 
     assert_eq!(

src/codec.rs

@@ -2,6 +2,12 @@ use bytemuck::{Pod, cast_slice};
 
 use crate::FastPForError;
 
+/// Internal default for max decompressed length. Used by trait defaults and C++ FFI.
+#[inline]
+pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize {
+    compressed_words.saturating_mul(1024)
+}
+
 /// Compresses and decompresses fixed-size blocks of `u32` values.
 ///
 /// The associated type [`Block`](BlockCodec::Block) is the concrete fixed-size
@@ -61,6 +67,17 @@ pub trait BlockCodec {
         n_blocks: usize,
         out: &mut Vec<u32>,
     ) -> Result<(), FastPForError>;
+
+    /// Maximum decompressed element count for a given compressed input length.
+    /// Reject `expected_len` values exceeding this to avoid allocation from bad data.
+    #[inline]
+    #[must_use]
+    fn max_decompressed_len(compressed_words: usize) -> usize
+    where
+        Self: Sized,
+    {
+        default_max_decoded_len(compressed_words)
+    }
 }
 
 /// Codec that supports compressing 64-bit integers into a 32-bit word stream.
@@ -89,8 +106,32 @@ pub trait AnyLenCodec {
     /// Compress an arbitrary-length slice of `u32` values.
     fn encode(&mut self, input: &[u32], out: &mut Vec<u32>) -> Result<(), FastPForError>;
 
+    /// Maximum decompressed element count for a given compressed input length.
+    /// Reject `expected_len` values exceeding this to avoid allocation from bad data.
+    #[inline]
+    #[must_use]
+    fn max_decompressed_len(compressed_words: usize) -> usize
+    where
+        Self: Sized,
+    {
+        default_max_decoded_len(compressed_words)
+    }
+
     /// Decompress a previously compressed slice of `u32` values.
-    fn decode(&mut self, input: &[u32], out: &mut Vec<u32>) -> Result<(), FastPForError>;
+    ///
+    /// When `expected_len` is `Some(n)`:
+    /// - Rejects if `n` exceeds [`max_decompressed_len`](AnyLenCodec::max_decompressed_len)(`input.len()`)
+    /// - Pre-allocates output capacity for `n` elements
+    /// - Returns [`DecodedCountMismatch`](crate::FastPForError::DecodedCountMismatch) if the
+    ///   actual decoded count differs from `n`
+    ///
+    /// The hint is not trusted: values from untrusted metadata are capped before use.
+    fn decode(
+        &mut self,
+        input: &[u32],
+        out: &mut Vec<u32>,
+        expected_len: Option<u32>,
+    ) -> Result<(), FastPForError>;
 }
 
 /// Split a flat `&[u32]` into `(&[Blocks::Block], &[u32])` without copying.