Skip to content

Commit 4a0ddb7

Browse files
nyuriknyurik
committed
chore(cpp): reorganize codec structure and improve error handling
First step of moving towards unified C++ and Rust model of codecs.
1 parent e5e9bf4 commit 4a0ddb7

28 files changed

Lines changed: 1076 additions & 810 deletions

‎Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,13 @@ cpp_portable = ["cpp"]
3838
# Optimize FastPFOR for the current CPU.
3939
cpp_native = ["cpp"]
4040
cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"]
41-
rust = ["dep:thiserror", "dep:bytes", "dep:bytemuck"]
41+
rust = ["dep:bytes"]
4242

4343
[dependencies]
44-
bytemuck = { version = "1.25.0", optional = true }
44+
bytemuck = { version = "1.25.0", features = ["min_const_generics"] }
4545
bytes = { version = "1.11", optional = true }
4646
cxx = { version = "1.0.194", optional = true }
47-
thiserror = { version = "2.0.18", optional = true }
47+
thiserror = "2.0.18"
4848

4949
[build-dependencies]
5050
cmake = { version = "0.1.57", optional = true }

‎README.md‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,21 +90,21 @@ Feature selection can be overridden with the `FASTPFOR_SIMD_MODE` environment va
9090
### Using C++ Wrapper
9191

9292
```rust
93-
use fastpfor::cpp::{Codec32 as _, SimdFastPFor128Codec};
93+
use fastpfor::{AnyLenCodec as _, cpp::SimdFastPFor128};
9494

9595
fn main() {
96-
let mut codec = SimdFastPFor128Codec::new();
96+
let mut codec = SimdFastPFor128::new();
9797

98-
// Encode
99-
let mut input = vec![1, 2, 3, 4, 5];
100-
let mut output = vec![0; 10]; // must be large enough
101-
let enc_slice = codec.encode32(&input, &mut output).unwrap();
98+
let input = vec![1u32, 2, 3, 4, 5];
99+
let mut compressed = Vec::new();
100+
codec.encode(&input, &mut compressed).unwrap();
102101

103-
// Decode
104-
let mut decoded = vec![0; 10]; // must be large enough
105-
let dec_slice = codec.decode32(&enc_slice, &mut decoded).unwrap();
102+
let mut decoded = Vec::new();
103+
codec
104+
.decode(&compressed, &mut decoded, None)
105+
.unwrap();
106106

107-
assert_eq!(input, dec_slice);
107+
assert_eq!(input, decoded);
108108
}
109109
```
110110

‎benches/bench_utils.rs‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ use core::ops::Range;
1313
pub use std::io::Cursor;
1414
use std::num::NonZeroU32;
1515

16+
#[cfg(feature = "cpp")]
17+
use fastpfor::AnyLenCodec as _;
18+
#[cfg(feature = "cpp")]
19+
use fastpfor::cpp;
1620
pub use fastpfor::rust::{BLOCK_SIZE_128, BLOCK_SIZE_256, DEFAULT_PAGE_SIZE, FastPFOR, Integer};
1721
use rand::rngs::StdRng;
1822
use rand::{RngExt as _, SeedableRng};
@@ -167,22 +171,24 @@ fn prepare_compressed_data(data: &[u32], block_size: NonZeroU32) -> Vec<u32> {
167171
// ---------------------------------------------------------------------------
168172

169173
#[cfg(feature = "cpp")]
170-
pub fn cpp_encode(codec: &fastpfor::cpp::FastPFor128Codec, data: &[u32]) -> Vec<u32> {
171-
use fastpfor::cpp::Codec32 as _;
172-
let mut out = vec![0u32; data.len() * 2 + 1024];
173-
let new_len = codec.encode32(data, &mut out).unwrap().len();
174-
out.truncate(new_len);
174+
pub fn cpp_encode(codec: &mut cpp::FastPFor128, data: &[u32]) -> Vec<u32> {
175+
let mut out = Vec::new();
176+
codec.encode(data, &mut out).unwrap();
175177
out
176178
}
177179

178180
#[cfg(feature = "cpp")]
179181
pub fn cpp_decode(
180-
codec: &fastpfor::cpp::FastPFor128Codec,
182+
codec: &mut cpp::FastPFor128,
181183
compressed: &[u32],
182184
decompressed: &mut [u32],
183185
) -> usize {
184-
use fastpfor::cpp::Codec32 as _;
185-
codec.decode32(compressed, decompressed).unwrap().len()
186+
let mut out = Vec::new();
187+
codec
188+
.decode(compressed, &mut out, Some(decompressed.len() as u32))
189+
.unwrap();
190+
decompressed.copy_from_slice(&out);
191+
out.len()
186192
}
187193

188194
// ---------------------------------------------------------------------------
@@ -268,10 +274,9 @@ pub struct CppDecodeFixture {
268274
#[cfg(feature = "cpp")]
269275
impl CppDecodeFixture {
270276
fn new(name: &'static str, generator: DataGeneratorFn, size: usize) -> Self {
271-
use fastpfor::cpp::FastPFor128Codec;
272277
let data = generator(size);
273-
let codec = FastPFor128Codec::new();
274-
let cpp_compressed = cpp_encode(&codec, &data);
278+
let mut codec = cpp::FastPFor128::new();
279+
let cpp_compressed = cpp_encode(&mut codec, &data);
275280
let rust_compressed = prepare_compressed_data(&data, BLOCK_SIZE_128);
276281
Self {
277282
name,

‎benches/fastpfor_benchmark.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ fn benchmark_compression_ratio(c: &mut Criterion) {
144144
/// the pure-Rust `FastPFOR` codec with `BLOCK_SIZE_128`.
145145
#[cfg(feature = "cpp")]
146146
fn benchmark_cpp_vs_rust(c: &mut Criterion) {
147-
use fastpfor::cpp::FastPFor128Codec;
147+
use fastpfor::cpp::FastPFor128;
148148

149149
let mut group = c.benchmark_group("cpp_vs_rust/encode");
150150
for (size, fix) in compress_fixtures(SIZES) {
@@ -153,8 +153,8 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
153153
BenchmarkId::new(format!("cpp/{}", fix.name), size),
154154
&fix.data,
155155
|b, data| {
156-
let codec = FastPFor128Codec::new();
157-
b.iter(|| black_box(cpp_encode(&codec, black_box(data))));
156+
let mut codec = FastPFor128::new();
157+
b.iter(|| black_box(cpp_encode(&mut codec, black_box(data))));
158158
},
159159
);
160160
group.bench_with_input(
@@ -175,9 +175,9 @@ fn benchmark_cpp_vs_rust(c: &mut Criterion) {
175175
BenchmarkId::new(format!("cpp/{}", fix.name), size),
176176
&fix.cpp_compressed,
177177
|b, compressed| {
178-
let codec = FastPFor128Codec::new();
178+
let mut codec = FastPFor128::new();
179179
let mut out = vec![0u32; fix.original_len];
180-
b.iter(|| black_box(cpp_decode(&codec, black_box(compressed), &mut out)));
180+
b.iter(|| black_box(cpp_decode(&mut codec, black_box(compressed), &mut out)));
181181
},
182182
);
183183
group.bench_with_input(

‎fuzz/fuzz_targets/common.rs‎

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
use fastpfor::{cpp, rust};
1+
use fastpfor::{AnyLenCodec, cpp, rust};
22

3-
pub type BoxedCppCodec = Box<dyn cpp::Codec32>;
3+
pub type BoxedCppCodec = Box<dyn AnyLenCodec>;
44

55
#[derive(arbitrary::Arbitrary)]
66
pub struct FuzzInput<C> {
@@ -77,41 +77,47 @@ pub enum CppCodec {
7777

7878
impl From<CppCodec> for BoxedCppCodec {
7979
fn from(codec: CppCodec) -> Self {
80-
use cpp::*;
80+
use cpp::{
81+
BP32, Copy as CppCopy, FastBinaryPacking8, FastBinaryPacking16, FastBinaryPacking32,
82+
FastPFor128, FastPFor256, MaskedVByte, NewPFor, OptPFor, PFor, PFor2008,
83+
SimdBinaryPacking, SimdFastPFor128, SimdFastPFor256, SimdGroupSimple,
84+
SimdGroupSimpleRingBuf, SimdNewPFor, SimdOptPFor, SimdPFor, SimdSimplePFor,
85+
StreamVByte, VByte, VarInt, VarIntGb,
86+
};
8187
match codec {
82-
CppCodec::BP32 => Box::new(BP32Codec::default()),
83-
CppCodec::Copy => Box::new(CopyCodec::default()),
84-
CppCodec::FastBinaryPacking8 => Box::new(FastBinaryPacking8Codec::default()),
85-
CppCodec::FastPFor128 => Box::new(FastPFor128Codec::default()),
86-
CppCodec::FastPFor256 => Box::new(FastPFor256Codec::default()),
87-
CppCodec::FastBinaryPacking16 => Box::new(FastBinaryPacking16Codec::default()),
88-
CppCodec::FastBinaryPacking32 => Box::new(FastBinaryPacking32Codec::default()),
89-
CppCodec::MaskedVByte => Box::new(MaskedVByteCodec::default()),
90-
CppCodec::NewPFor => Box::new(NewPForCodec::default()),
91-
CppCodec::OptPFor => Box::new(OptPForCodec::default()),
92-
CppCodec::PFor2008 => Box::new(PFor2008Codec::default()),
93-
CppCodec::PFor => Box::new(PForCodec::default()),
94-
CppCodec::SimdBinaryPacking => Box::new(SimdBinaryPackingCodec::default()),
95-
CppCodec::SimdFastPFor128 => Box::new(SimdFastPFor128Codec::default()),
96-
CppCodec::SimdFastPFor256 => Box::new(SimdFastPFor256Codec::default()),
97-
CppCodec::SimdGroupSimple => Box::new(SimdGroupSimpleCodec::default()),
98-
CppCodec::SimdGroupSimpleRingBuf => Box::new(SimdGroupSimpleRingBufCodec::default()),
99-
CppCodec::SimdNewPFor => Box::new(SimdNewPForCodec::default()),
100-
CppCodec::SimdOptPFor => Box::new(SimdOptPForCodec::default()),
101-
CppCodec::SimdPFor => Box::new(SimdPForCodec::default()),
102-
CppCodec::SimdSimplePFor => Box::new(SimdSimplePForCodec::default()),
88+
CppCodec::BP32 => Box::new(BP32::default()),
89+
CppCodec::Copy => Box::new(CppCopy::default()),
90+
CppCodec::FastBinaryPacking8 => Box::new(FastBinaryPacking8::default()),
91+
CppCodec::FastPFor128 => Box::new(FastPFor128::default()),
92+
CppCodec::FastPFor256 => Box::new(FastPFor256::default()),
93+
CppCodec::FastBinaryPacking16 => Box::new(FastBinaryPacking16::default()),
94+
CppCodec::FastBinaryPacking32 => Box::new(FastBinaryPacking32::default()),
95+
CppCodec::MaskedVByte => Box::new(MaskedVByte::default()),
96+
CppCodec::NewPFor => Box::new(NewPFor::default()),
97+
CppCodec::OptPFor => Box::new(OptPFor::default()),
98+
CppCodec::PFor2008 => Box::new(PFor2008::default()),
99+
CppCodec::PFor => Box::new(PFor::default()),
100+
CppCodec::SimdBinaryPacking => Box::new(SimdBinaryPacking::default()),
101+
CppCodec::SimdFastPFor128 => Box::new(SimdFastPFor128::default()),
102+
CppCodec::SimdFastPFor256 => Box::new(SimdFastPFor256::default()),
103+
CppCodec::SimdGroupSimple => Box::new(SimdGroupSimple::default()),
104+
CppCodec::SimdGroupSimpleRingBuf => Box::new(SimdGroupSimpleRingBuf::default()),
105+
CppCodec::SimdNewPFor => Box::new(SimdNewPFor::default()),
106+
CppCodec::SimdOptPFor => Box::new(SimdOptPFor::default()),
107+
CppCodec::SimdPFor => Box::new(SimdPFor::default()),
108+
CppCodec::SimdSimplePFor => Box::new(SimdSimplePFor::default()),
103109
// CppCodec::Simple16 => Box::new(Simple16Codec::default()),
104110
// CppCodec::Simple8b => Box::new(Simple8bCodec::default()),
105111
// CppCodec::Simple8bRle => Box::new(Simple8bRleCodec::default()),
106112
// CppCodec::Simple9 => Box::new(Simple9Codec::default()),
107113
// CppCodec::Simple9Rle => Box::new(Simple9RleCodec::default()),
108114
// CppCodec::SimplePFor => Box::new(SimplePForCodec::default()),
109115
// CppCodec::Snappy => Box::new(SnappyCodec::default()),
110-
CppCodec::StreamVByte => Box::new(StreamVByteCodec::default()),
111-
CppCodec::VByte => Box::new(VByteCodec::default()),
112-
CppCodec::VarInt => Box::new(VarIntCodec::default()),
116+
CppCodec::StreamVByte => Box::new(StreamVByte::default()),
117+
CppCodec::VByte => Box::new(VByte::default()),
118+
CppCodec::VarInt => Box::new(VarInt::default()),
113119
// CppCodec::VarIntG8iu => Box::new(VarIntG8iuCodec::default()),
114-
CppCodec::VarIntGb => Box::new(VarIntGbCodec::default()),
120+
CppCodec::VarIntGb => Box::new(VarIntGb::default()),
115121
// CppCodec::VsEncoding => Box::new(VsEncodingCodec::default()),
116122
}
117123
}

‎fuzz/fuzz_targets/cpp_roundtrip.rs‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,26 @@ mod common;
55
use common::*;
66

77
fuzz_target!(|data: FuzzInput<CppCodec>| {
8-
let codec = BoxedCppCodec::from(data.codec);
8+
let mut codec = BoxedCppCodec::from(data.codec);
99
let input = data.data;
1010

11-
// Allocate output buffer with generous size
12-
let mut output = vec![0u32; input.len() * 2 + 1024];
11+
let mut compressed = Vec::new();
12+
codec.encode(&input, &mut compressed).unwrap();
1313

14-
// Compress the data
15-
let enc_slice = codec.encode32(&input, &mut output).unwrap();
16-
17-
// Now decompress
18-
let mut decoded = vec![0u32; input.len() * 2 + 1024];
19-
let dec_slice = codec.decode32(enc_slice, &mut decoded).unwrap();
14+
let mut decoded = Vec::new();
15+
codec
16+
.decode(&compressed, &mut decoded, None)
17+
.expect("decode");
2018

2119
// Verify roundtrip
22-
if dec_slice.len() + input.len() < 200 {
23-
assert_eq!(input, dec_slice, "Decompressed output mismatches");
20+
if decoded.len() + input.len() < 200 {
21+
assert_eq!(input, decoded.as_slice(), "Decompressed output mismatches");
2422
} else {
25-
assert_eq!(dec_slice.len(), input.len(), "Decompressed length mismatch");
26-
for (i, (&original, &decoded)) in input.iter().zip(dec_slice.iter()).enumerate() {
23+
assert_eq!(decoded.len(), input.len(), "Decompressed length mismatch");
24+
for (i, (&original, &out)) in input.iter().zip(decoded.iter()).enumerate() {
2725
assert_eq!(
28-
original, decoded,
29-
"Mismatch at position {i}: expected {original}, got {decoded}"
26+
original, out,
27+
"Mismatch at position {i}: expected {original}, got {out}"
3028
);
3129
}
3230
}

‎fuzz/fuzz_targets/rust_compress_oracle.rs‎

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#![no_main]
22

3-
use fastpfor::{CodecToSlice, cpp, rust};
3+
use fastpfor::{AnyLenCodec, CodecToSlice, cpp, rust};
44
use libfuzzer_sys::fuzz_target;
55
mod common;
66
use common::*;
@@ -28,43 +28,32 @@ fuzz_target!(|data: FuzzInput<RustCodec>| {
2828
let last_block_size_multiple = input.len() / block_size * block_size;
2929
let input = &input[..last_block_size_multiple];
3030

31-
// Allocate output buffers with generous size
31+
// Allocate output buffer for Rust (slice API)
3232
let mut rust_compressed = vec![0u32; input.len() * 2 + 1024];
33-
let mut cpp_compressed = vec![0u32; input.len() * 2 + 1024];
3433

3534
// Compress with Rust implementation using Codec wrapper
3635
let mut rust_codec = rust::Codec::from(data.codec);
3736
let rust_result = rust_codec
3837
.compress_to_slice(input, &mut rust_compressed)
3938
.expect("Rust compression failed");
4039

41-
// Compress with C++ implementation
42-
let compressed_oracle_from_cpp = match data.codec {
43-
RustCodec::FastPFOR256 => {
44-
let mut cpp_codec = cpp::FastPFor256Codec::new();
45-
cpp_codec
46-
.compress_to_slice(input, &mut cpp_compressed)
47-
.expect("C++ compression failed")
48-
}
49-
RustCodec::FastPFOR128 => {
50-
let mut cpp_codec = cpp::FastPFor128Codec::new();
51-
cpp_codec
52-
.compress_to_slice(input, &mut cpp_compressed)
53-
.expect("C++ compression failed")
54-
}
55-
RustCodec::VariableByte => {
56-
let mut cpp_codec = cpp::MaskedVByteCodec::new();
57-
cpp_codec
58-
.compress_to_slice(input, &mut cpp_compressed)
59-
.expect("C++ compression failed")
60-
}
61-
RustCodec::JustCopy => {
62-
let mut cpp_codec = cpp::CopyCodec::new();
63-
cpp_codec
64-
.compress_to_slice(input, &mut cpp_compressed)
65-
.expect("C++ compression failed")
66-
}
67-
};
40+
// Compress with C++ implementation (`AnyLenCodec` / Vec API)
41+
let mut cpp_compressed = Vec::new();
42+
match data.codec {
43+
RustCodec::FastPFOR256 => cpp::FastPFor256::new()
44+
.encode(input, &mut cpp_compressed)
45+
.expect("C++ compression failed"),
46+
RustCodec::FastPFOR128 => cpp::FastPFor128::new()
47+
.encode(input, &mut cpp_compressed)
48+
.expect("C++ compression failed"),
49+
RustCodec::VariableByte => cpp::MaskedVByte::new()
50+
.encode(input, &mut cpp_compressed)
51+
.expect("C++ compression failed"),
52+
RustCodec::JustCopy => cpp::Copy::new()
53+
.encode(input, &mut cpp_compressed)
54+
.expect("C++ compression failed"),
55+
}
56+
let compressed_oracle_from_cpp = cpp_compressed.as_slice();
6857

6958
// Compare compressed outputs
7059
assert_eq!(

‎fuzz/fuzz_targets/rust_decompress_oracle.rs‎

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#![no_main]
22

3-
use fastpfor::{CodecToSlice, cpp, rust};
3+
use fastpfor::{AnyLenCodec, CodecToSlice, cpp, rust};
44
use libfuzzer_sys::fuzz_target;
55
mod common;
66
use common::*;
@@ -29,33 +29,22 @@ fuzz_target!(|data: FuzzInput<RustCodec>| {
2929
let input = &input[..last_block_size_multiple];
3030

3131
// First, compress with C++ implementation to get valid compressed data
32-
let mut cpp_compressed = vec![0u32; input.len() * 2 + 1024];
33-
let compressed_oracle_from_cpp = match data.codec {
34-
RustCodec::FastPFOR256 => {
35-
let mut cpp_codec = cpp::FastPFor256Codec::new();
36-
cpp_codec
37-
.compress_to_slice(input, &mut cpp_compressed)
38-
.expect("C++ compression failed")
39-
}
40-
RustCodec::FastPFOR128 => {
41-
let mut cpp_codec = cpp::FastPFor128Codec::new();
42-
cpp_codec
43-
.compress_to_slice(input, &mut cpp_compressed)
44-
.expect("C++ compression failed")
45-
}
46-
RustCodec::VariableByte => {
47-
let mut cpp_codec = cpp::MaskedVByteCodec::new();
48-
cpp_codec
49-
.compress_to_slice(input, &mut cpp_compressed)
50-
.expect("C++ compression failed")
51-
}
52-
RustCodec::JustCopy => {
53-
let mut cpp_codec = cpp::CopyCodec::new();
54-
cpp_codec
55-
.compress_to_slice(input, &mut cpp_compressed)
56-
.expect("C++ compression failed")
57-
}
58-
};
32+
let mut cpp_compressed = Vec::new();
33+
match data.codec {
34+
RustCodec::FastPFOR256 => cpp::FastPFor256::new()
35+
.encode(input, &mut cpp_compressed)
36+
.expect("C++ compression failed"),
37+
RustCodec::FastPFOR128 => cpp::FastPFor128::new()
38+
.encode(input, &mut cpp_compressed)
39+
.expect("C++ compression failed"),
40+
RustCodec::VariableByte => cpp::MaskedVByte::new()
41+
.encode(input, &mut cpp_compressed)
42+
.expect("C++ compression failed"),
43+
RustCodec::JustCopy => cpp::Copy::new()
44+
.encode(input, &mut cpp_compressed)
45+
.expect("C++ compression failed"),
46+
}
47+
let compressed_oracle_from_cpp = cpp_compressed.as_slice();
5948

6049
// Now decompress with rust
6150
let mut rust_decompressed = vec![0u32; input.len()];

0 commit comments

Comments
 (0)