diff --git a/src/cpp/mod.rs b/src/cpp/mod.rs index e145ab0..4855679 100644 --- a/src/cpp/mod.rs +++ b/src/cpp/mod.rs @@ -4,6 +4,8 @@ pub use cxx::Exception; use cxx::UniquePtr; +use crate::CodecToSlice; + /// FFI bridge to the C++ FastPFOR library. /// /// This module contains the raw FFI declarations for interfacing with the C++ code. @@ -142,6 +144,51 @@ pub trait Codec32: CodecWrapper { } } +impl CodecToSlice for C { + type Error = Exception; + + fn compress_to_slice<'out>( + &mut self, + input: &[u32], + output: &'out mut [u32], + ) -> Result<&'out [u32], Self::Error> { + let result = self.encode32(input, output)?; + Ok(result) + } + + fn decompress_to_slice<'out>( + &mut self, + input: &[u32], + output: &'out mut [u32], + ) -> Result<&'out [u32], Self::Error> { + let result = self.decode32(input, output)?; + Ok(result) + } +} + +// Note: 64-bit integers are compressed into 32-bit word arrays. +impl CodecToSlice for C { + type Error = Exception; + + fn compress_to_slice<'out>( + &mut self, + input: &[u64], + output: &'out mut [u32], + ) -> Result<&'out [u32], Self::Error> { + let result = self.encode64(input, output)?; + Ok(result) + } + + fn decompress_to_slice<'out>( + &mut self, + input: &[u32], + output: &'out mut [u64], + ) -> Result<&'out [u64], Self::Error> { + let result = self.decode64(input, output)?; + Ok(result) + } +} + /// Trait for codecs that support 64-bit integer compression. /// /// Only certain codecs support 64-bit integers. These are marked with the `@ 64` @@ -385,38 +432,50 @@ mod tests { #[test] fn test_32() { - let codec = FastPFor128Codec::new(); + let mut codec = FastPFor128Codec::new(); let input = vec![1, 2, 3, 4, 5]; let mut output = vec![0; 10]; let mut output2 = vec![0; 10]; + let mut output3 = vec![0; 10]; let encoded = codec.encode32(&input, &mut output).unwrap(); let encoded2 = codec.encode32(&input, &mut output2).unwrap(); + let encoded3 = codec.compress_to_slice(&input, &mut output3).unwrap(); assert_eq!(encoded, encoded2); + assert_eq!(encoded, encoded3); let mut decoded = vec![0; 10]; let mut decoded2 = vec![0; 10]; + let mut decoded3 = vec![0; 10]; let decoded = codec.decode32(encoded, &mut decoded).unwrap(); let decoded2 = codec.decode32(encoded, &mut decoded2).unwrap(); + let decoded3 = codec.decompress_to_slice(encoded, &mut decoded3).unwrap(); assert_eq!(decoded, decoded2); + assert_eq!(decoded, decoded3); assert_eq!(decoded, input); } #[test] fn test_64() { - let codec = FastPFor128Codec::new(); + let mut codec = FastPFor128Codec::new(); let input = vec![1, 2, 3, 4, 5]; let mut output = vec![0; 10]; let mut output2 = vec![0; 10]; + let mut output3 = vec![0; 10]; let encoded = codec.encode64(&input, &mut output).unwrap(); let encoded2 = codec.encode64(&input, &mut output2).unwrap(); + let encoded3 = codec.compress_to_slice(&input, &mut output3).unwrap(); assert_eq!(encoded, encoded2); + assert_eq!(encoded, encoded3); let mut decoded = vec![0; 10]; let mut decoded2 = vec![0; 10]; + let mut decoded3 = vec![0; 10]; let decoded = codec.decode64(encoded, &mut decoded).unwrap(); let decoded2 = codec.decode64(encoded, &mut decoded2).unwrap(); + let decoded3 = codec.decompress_to_slice(encoded, &mut decoded3).unwrap(); assert_eq!(decoded, decoded2); + assert_eq!(decoded, decoded3); assert_eq!(decoded, input); } diff --git a/src/lib.rs b/src/lib.rs index 92afc0f..6cadd12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,3 +16,39 @@ pub mod cpp; #[cfg(feature = "rust")] /// Rust re-implementation of `FastPFor` (work in progress) pub mod rust; + +/// Low-level compression interface using caller-provided buffers. +/// +/// Codecs write into pre-allocated slices and return a sub-slice showing exactly +/// what was written. Works across FFI boundaries and allows buffer reuse. +/// +/// # Type Parameters +/// +/// - `In`: Input data type (e.g., `u32` or `u64` for integer codecs) +/// - `Out`: Compressed output type (defaults to `In`, but may differ - e.g., +/// 64-bit integers compress to 32-bit words: `CodecToSlice`) +/// +/// # Buffer Sizing +/// +/// Caller must ensure output buffers are large enough. For compression, estimate +/// `input.len() * 2 + 1024`. For decompression, size depends on the codec. +pub trait CodecToSlice { + /// Error type returned by compression/decompression operations. + type Error; + + /// Compresses input into output buffer, returning slice of data written. + fn compress_to_slice<'out>( + &mut self, + input: &[In], + output: &'out mut [Out], + ) -> Result<&'out [Out], Self::Error>; + + /// Decompresses input into output buffer, returning slice of data written. + /// + /// Output size cannot be known in advance for some codecs (e.g., RLE). + fn decompress_to_slice<'out>( + &mut self, + input: &[Out], + output: &'out mut [In], + ) -> Result<&'out [In], Self::Error>; +} diff --git a/src/rust/error.rs b/src/rust/error.rs index 440d7e9..23964ed 100644 --- a/src/rust/error.rs +++ b/src/rust/error.rs @@ -18,4 +18,8 @@ pub enum FastPForError { /// Output buffer too small #[error("Output buffer too small")] OutputBufferTooSmall, + + /// Invalid input length + #[error("Invalid input length {0}")] + InvalidInputLength(usize), } diff --git a/src/rust/integer_compression/codec.rs b/src/rust/integer_compression/codec.rs index 59de19e..67ced9d 100644 --- a/src/rust/integer_compression/codec.rs +++ b/src/rust/integer_compression/codec.rs @@ -1,4 +1,7 @@ -use crate::rust::{FastPFOR, JustCopy, VariableByte}; +use std::io::Cursor; + +use crate::rust::{FastPFOR, FastPForResult, Integer, JustCopy, VariableByte}; +use crate::CodecToSlice; /// Type-erased wrapper for compression codecs. /// @@ -12,6 +15,100 @@ pub enum Codec { JustCopy(JustCopy), } +impl Integer for Codec { + fn compress( + &mut self, + input: &[u32], + input_length: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) -> FastPForResult<()> { + match self { + Codec::FastPFor(fastpfor) => { + fastpfor.compress(input, input_length, input_offset, output, output_offset) + } + Codec::VariableByte(vb) => { + vb.compress(input, input_length, input_offset, output, output_offset) + } + Codec::JustCopy(jc) => { + jc.compress(input, input_length, input_offset, output, output_offset) + } + } + } + + fn uncompress( + &mut self, + input: &[u32], + input_length: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) -> FastPForResult<()> { + match self { + Codec::FastPFor(fastpfor) => { + fastpfor.uncompress(input, input_length, input_offset, output, output_offset) + } + Codec::VariableByte(vb) => { + vb.uncompress(input, input_length, input_offset, output, output_offset) + } + Codec::JustCopy(jc) => { + jc.uncompress(input, input_length, input_offset, output, output_offset) + } + } + } +} + +impl CodecToSlice for Codec { + type Error = crate::rust::FastPForError; + + fn compress_to_slice<'out>( + &mut self, + input: &[u32], + output: &'out mut [u32], + ) -> Result<&'out [u32], Self::Error> { + let mut output_offset = Cursor::new(0); + let input_length = input + .len() + .try_into() + .map_err(|_| Self::Error::InvalidInputLength(input.len()))?; + + self.compress( + input, + input_length, + &mut Cursor::new(0), + output, + &mut output_offset, + )?; + + let written = output_offset.position() as usize; + Ok(&output[..written]) + } + + fn decompress_to_slice<'out>( + &mut self, + input: &[u32], + output: &'out mut [u32], + ) -> Result<&'out [u32], Self::Error> { + let mut output_offset = Cursor::new(0); + let input_length: u32 = input + .len() + .try_into() + .map_err(|_| Self::Error::InvalidInputLength(input.len()))?; + + self.uncompress( + input, + input_length, + &mut Cursor::new(0), + output, + &mut output_offset, + )?; + + let written = output_offset.position() as usize; + Ok(&output[..written]) + } +} + impl From for Codec { fn from(fastpfor: FastPFOR) -> Self { Codec::FastPFor(Box::new(fastpfor)) @@ -29,3 +126,28 @@ impl From for Codec { Codec::JustCopy(jc) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supports_compress_to_slice() { + let data = vec![1, 2, 3, 4, 5]; + let mut rust_codec = Codec::from(VariableByte::new()); + let mut compressed = vec![0u32; data.len() * 4]; + + let compressed_len = { + let result = rust_codec + .compress_to_slice(&data, &mut compressed) + .unwrap(); + result.len() + }; + + let mut decompressed = vec![0u32; data.len()]; + let result = rust_codec + .decompress_to_slice(&compressed[..compressed_len], &mut decompressed) + .unwrap(); + assert_eq!(result, &data[..]); + } +} diff --git a/src/rust/integer_compression/integer_codec.rs b/src/rust/integer_compression/integer_codec.rs index afceb40..4ab11fd 100644 --- a/src/rust/integer_compression/integer_codec.rs +++ b/src/rust/integer_compression/integer_codec.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use crate::rust::{Codec, FastPForResult}; +use crate::rust::FastPForResult; /// Integer compression/decompression interface with length headers. /// @@ -37,47 +37,3 @@ pub trait Integer { output_offset: &mut Cursor, ) -> FastPForResult<()>; } - -impl Integer for Codec { - fn compress( - &mut self, - input: &[u32], - input_length: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) -> FastPForResult<()> { - match self { - Codec::FastPFor(fastpfor) => { - fastpfor.compress(input, input_length, input_offset, output, output_offset) - } - Codec::VariableByte(vb) => { - vb.compress(input, input_length, input_offset, output, output_offset) - } - Codec::JustCopy(jc) => { - jc.compress(input, input_length, input_offset, output, output_offset) - } - } - } - - fn uncompress( - &mut self, - input: &[u32], - input_length: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) -> FastPForResult<()> { - match self { - Codec::FastPFor(fastpfor) => { - fastpfor.uncompress(input, input_length, input_offset, output, output_offset) - } - Codec::VariableByte(vb) => { - vb.uncompress(input, input_length, input_offset, output, output_offset) - } - Codec::JustCopy(jc) => { - jc.uncompress(input, input_length, input_offset, output, output_offset) - } - } - } -}