Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@ Unless otherwise specified, all codecs support `&[u32]` only.
```text
* BP32
* Copy
* FastBinaryPacking16
* FastBinaryPacking32
* FastBinaryPacking8
* FastPFor128 (both `&[u32]` and `&[u64]`)
* FastPFor256 (both `&[u32]` and `&[u64]`)
* FastBinaryPacking16
* FastBinaryPacking32
* MaskedVByte
* NewPFor
* OptPFor
* PFor2008
* PFor
* PFor2008
* SimdBinaryPacking
* SimdFastPFor128
* SimdFastPFor256
Expand Down
33 changes: 16 additions & 17 deletions src/codec.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use bytemuck::{Pod, cast_slice};

use crate::FastPForError;
use crate::FastPForResult;

/// Internal default for max decompressed length. Used by trait defaults and C++ FFI.
#[inline]
Expand All @@ -23,13 +23,15 @@ pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize {
///
/// # Implementing this trait
///
/// ```rust,ignore
/// ```
/// # use fastpfor::{BlockCodec, FastPForResult};
/// struct MyCodec;
/// impl BlockCodec for MyCodec {
/// type Block = [u32; 256];
/// fn encode_blocks(&self, blocks: &[[u32; 256]], out: &mut Vec<u32>)
/// -> Result<(), FastPForError> { ... }
/// fn decode_blocks(&self, input: &[u32], expected_len: Option<u32>,
/// out: &mut Vec<u32>) -> Result<usize, FastPForError> { ... }
/// fn encode_blocks(&mut self, blocks: &[[u32; 256]], out: &mut Vec<u32>)
/// -> FastPForResult<()> { todo!() }
/// fn decode_blocks(&mut self, input: &[u32], expected_len: Option<u32>,
/// out: &mut Vec<u32>) -> FastPForResult<usize> { todo!() }
/// }
/// ```
pub trait BlockCodec {
Expand All @@ -54,11 +56,7 @@ pub trait BlockCodec {
///
/// No remainder is possible — the caller must split the input first using
/// [`slice_to_blocks`] and handle any remainder separately.
fn encode_blocks(
&mut self,
blocks: &[Self::Block],
out: &mut Vec<u32>,
) -> Result<(), FastPForError>;
fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec<u32>) -> FastPForResult<()>;

/// Decompress blocks from `input`, using the length stored in the header.
///
Expand All @@ -75,7 +73,7 @@ pub trait BlockCodec {
input: &[u32],
expected_len: Option<u32>,
out: &mut Vec<u32>,
) -> Result<usize, FastPForError>;
) -> FastPForResult<usize>;

/// Maximum decompressed element count for a given compressed input length.
/// Reject `expected_len` values exceeding this to avoid allocation from bad data.
Expand All @@ -100,9 +98,9 @@ pub trait BlockCodec {
#[cfg(feature = "cpp")]
pub trait BlockCodec64 {
/// Compress 64-bit integers into a 32-bit word stream.
fn encode64(&mut self, input: &[u64], out: &mut Vec<u32>) -> Result<(), FastPForError>;
fn encode64(&mut self, input: &[u64], out: &mut Vec<u32>) -> FastPForResult<()>;
/// Decompress 64-bit integers from a 32-bit word stream.
fn decode64(&mut self, input: &[u32], out: &mut Vec<u64>) -> Result<(), FastPForError>;
fn decode64(&mut self, input: &[u32], out: &mut Vec<u64>) -> FastPForResult<()>;
}

/// Compresses and decompresses an arbitrary-length `&[u32]` slice.
Expand All @@ -113,7 +111,7 @@ pub trait BlockCodec64 {
/// to produce an `AnyLenCodec`.
pub trait AnyLenCodec {
/// Compress an arbitrary-length slice of `u32` values.
fn encode(&mut self, input: &[u32], out: &mut Vec<u32>) -> Result<(), FastPForError>;
fn encode(&mut self, input: &[u32], out: &mut Vec<u32>) -> FastPForResult<()>;

/// Maximum decompressed element count for a given compressed input length.
/// Reject `expected_len` values exceeding this to avoid allocation from bad data.
Expand All @@ -140,7 +138,7 @@ pub trait AnyLenCodec {
input: &[u32],
out: &mut Vec<u32>,
expected_len: Option<u32>,
) -> Result<(), FastPForError>;
) -> FastPForResult<()>;
}

/// Split a flat `&[u32]` into `(&[Blocks::Block], &[u32])` without copying.
Expand All @@ -154,7 +152,8 @@ pub trait AnyLenCodec {
///
/// # Example
///
/// ```rust,ignore
/// ```ignore
/// # use fastpfor::{slice_to_blocks, FastPForBlock256};
/// let data: Vec<u32> = (0..600).collect(); // 2 × 256 + 88 remainder
/// let (blocks, remainder) = slice_to_blocks::<FastPForBlock256>(&data);
/// assert_eq!(blocks.len(), 2); // 2 blocks of [u32; 256]
Expand Down
12 changes: 6 additions & 6 deletions src/cpp/codecs.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use cxx::UniquePtr;

use crate::FastPForError;
use crate::FastPForResult;
use crate::codec::{AnyLenCodec, BlockCodec64};
use crate::cpp::ffi;
use crate::cpp::wrappers::{
Expand Down Expand Up @@ -37,7 +37,7 @@ macro_rules! implement_cpp_codecs {
}

impl AnyLenCodec for $name {
fn encode(&mut self, input: &[u32], out: &mut Vec<u32>) -> Result<(), FastPForError> {
fn encode(&mut self, input: &[u32], out: &mut Vec<u32>) -> FastPForResult<()> {
encode32_to_vec_ffi(&self.0, input, out)
}

Expand All @@ -46,7 +46,7 @@ macro_rules! implement_cpp_codecs {
input: &[u32],
out: &mut Vec<u32>,
expected_len: Option<u32>,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
decode32_anylen_ffi(&self.0, input, out, expected_len)
}
}
Expand Down Expand Up @@ -161,10 +161,10 @@ macro_rules! implement_cpp_codecs_64 {
($($name:ident => $ffi:ident ,)*) => {
$(
impl BlockCodec64 for $name {
fn encode64(&mut self, input: &[u64], out: &mut Vec<u32>) -> Result<(), FastPForError> {
fn encode64(&mut self, input: &[u64], out: &mut Vec<u32>) -> FastPForResult<()> {
encode64_to_vec_ffi(&self.0, input, out)
}
fn decode64(&mut self, input: &[u32], out: &mut Vec<u64>) -> Result<(), FastPForError> {
fn decode64(&mut self, input: &[u32], out: &mut Vec<u64>) -> FastPForResult<()> {
decode64_to_vec_ffi(&self.0, input, out)
}
}
Expand Down Expand Up @@ -192,7 +192,7 @@ pub(crate) mod tests {
}

/// C++ `fastpfor256_codec` returns `CompositeCodec<FastPFor<8>, VariableByte>` — already
/// any-length. Use it directly; do not wrap in Rust `CppComposite`.
/// any-length. Use it directly; do not wrap in Rust `CompositeCodec`.

Copilot AI Mar 22, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test comment mentions wrapping in Rust CompositeCodec, but there is no CompositeCodec type in this crate (searching the source only finds references in docs/comments). Consider rewording to avoid naming a non-existent Rust wrapper type (e.g., “do not wrap it again on the Rust side; it already supports any-length input”).

Suggested change
/// any-length. Use it directly; do not wrap in Rust `CompositeCodec`.
/// any-length. Use it directly; do not wrap it again on the Rust side; it already supports any-length input.

Copilot uses AI. Check for mistakes.
#[test]
fn test_cpp_fastpfor256_composite_anylen() {
let mut codec = CppFastPFor256::new();
Expand Down
12 changes: 6 additions & 6 deletions src/cpp/wrappers.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use cxx::UniquePtr;

use crate::FastPForError;
use crate::FastPForResult;
use crate::codec::default_max_decoded_len;
use crate::cpp::ffi;
use crate::helpers::AsUsize;
Expand All @@ -14,7 +14,7 @@ pub fn encode32_to_vec_ffi(
codec: &UniquePtr<ffi::IntegerCODEC>,
input: &[u32],
out: &mut Vec<u32>,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
let capacity = input.len() * 2 + 1024;
let start = out.len();
out.resize(start + capacity, 0);
Expand All @@ -28,7 +28,7 @@ fn decode32_to_vec_ffi(
input: &[u32],
out: &mut Vec<u32>,
capacity: usize,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
if !input.is_empty() {
let start = out.len();
out.resize(start + capacity, 0);
Expand All @@ -43,7 +43,7 @@ pub fn decode32_anylen_ffi(
input: &[u32],
out: &mut Vec<u32>,
expected_len: Option<u32>,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
let max = default_max_decoded_len(input.len());
let capacity = if let Some(n) = expected_len {
n.is_valid_expected(max)?
Expand All @@ -64,7 +64,7 @@ pub fn encode64_to_vec_ffi(
codec: &UniquePtr<ffi::IntegerCODEC>,
input: &[u64],
out: &mut Vec<u32>,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
let capacity = input.len() * 3 + 1024;
let start = out.len();
out.resize(start + capacity, 0);
Expand All @@ -77,7 +77,7 @@ pub fn decode64_to_vec_ffi(
codec: &UniquePtr<ffi::IntegerCODEC>,
input: &[u32],
out: &mut Vec<u64>,
) -> Result<(), FastPForError> {
) -> FastPForResult<()> {
if !input.is_empty() {
// C++ decodeArray needs output buffer. Variable-byte can pack multiple values per word.
let capacity = input.len().saturating_mul(4);
Expand Down
12 changes: 6 additions & 6 deletions src/helpers.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::FastPForError;
use crate::{FastPForError, FastPForResult};

/// Finds the greatest multiple of `factor` that is less than or equal to `value`.
#[cfg_attr(feature = "cpp", allow(dead_code))]
Expand All @@ -18,7 +18,7 @@ pub trait AsUsize: Eq + Copy {

#[inline]
#[cfg(feature = "cpp")]
fn is_decoded_mismatch(self, expected: impl AsUsize) -> Result<(), FastPForError> {
fn is_decoded_mismatch(self, expected: impl AsUsize) -> FastPForResult<()> {
let actual = self.as_usize();
let expected = expected.as_usize();
if self.as_usize() == expected {
Expand All @@ -31,7 +31,7 @@ pub trait AsUsize: Eq + Copy {
/// Returns an error if `expected` exceeds `max`.
#[inline]
#[cfg(feature = "cpp")]
fn is_valid_expected(self, max: impl AsUsize) -> Result<usize, FastPForError> {
fn is_valid_expected(self, max: impl AsUsize) -> FastPForResult<usize> {
let expected = self.as_usize();
let max = max.as_usize();
if expected > max {
Expand Down Expand Up @@ -69,12 +69,12 @@ impl AsUsize for u32 {

#[cfg_attr(feature = "cpp", allow(dead_code))]
pub trait GetWithErr<T> {
fn get_val(&self, pos: impl AsUsize) -> Result<T, FastPForError>;
fn get_val(&self, pos: impl AsUsize) -> FastPForResult<T>;
}

impl<T: Copy> GetWithErr<T> for &[T] {
#[inline]
fn get_val(&self, pos: impl AsUsize) -> Result<T, FastPForError> {
fn get_val(&self, pos: impl AsUsize) -> FastPForResult<T> {
self.get(pos.as_usize())
.copied()
.ok_or(FastPForError::NotEnoughData)
Expand All @@ -83,7 +83,7 @@ impl<T: Copy> GetWithErr<T> for &[T] {

impl<T: Copy> GetWithErr<T> for Vec<T> {
#[inline]
fn get_val(&self, pos: impl AsUsize) -> Result<T, FastPForError> {
fn get_val(&self, pos: impl AsUsize) -> FastPForResult<T> {
self.as_slice().get_val(pos)
}
}
7 changes: 1 addition & 6 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,13 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
#![doc = include_str!("../README.md")]

#[cfg(not(any(feature = "cpp", feature = "rust",)))]
#[cfg(not(any(feature = "cpp", feature = "rust")))]
compile_error!("At least one of the features 'cpp' or 'rust' must be enabled");

// Error types are always available regardless of which codec features are enabled.
mod error;
pub use error::{FastPForError, FastPForResult};

// FIXME: need decide on the external API. Some ideas:
// - offer two sets of similar APIs - rust and cpp ffi
// - it will be possible to enable/disable each with a feature flag
// - introduce a new feature-agnostic API that will forward to either
// - if both are enabled, forward to the more stable (ffi probably)
#[cfg(feature = "cpp")]
/// Rust wrapper for the [`FastPFOR` C++ library](https://github.com/fast-pack/FastPFor)
pub mod cpp;
Expand Down
19 changes: 10 additions & 9 deletions src/rust/integer_compression/fastpfor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::array;
use std::io::Cursor;
use std::num::NonZeroU32;

use bytemuck::cast_slice;
use bytes::{Buf as _, BufMut as _, BytesMut};

use crate::helpers::{GetWithErr, bits, greatest_multiple};
Expand Down Expand Up @@ -176,13 +177,13 @@ impl FastPFOR {
/// - Writes header, packed data, metadata bytes, and exception values.
///
/// # Arguments
/// * `thissize` - Must be multiple of `block_size`
/// * `input_offset` - Advanced by `thissize`
/// * `this_size` - Must be multiple of `block_size`
/// * `input_offset` - Advanced by `this_size`
/// * `output_offset` - Advanced by compressed size
fn encode_page(
&mut self,
input: &[u32],
thissize: u32,
this_size: u32,
input_offset: &mut Cursor<u32>,
output: &mut [u32],
output_offset: &mut Cursor<u32>,
Expand All @@ -196,7 +197,7 @@ impl FastPFOR {
self.bytes_container.clear();

let mut tmp_input_offset = input_offset.position() as u32;
let final_input_offset = tmp_input_offset + thissize - self.block_size;
let final_input_offset = tmp_input_offset + this_size - self.block_size;
while tmp_input_offset <= final_input_offset {
self.best_bit_from_data(input, tmp_input_offset);
self.bytes_container.put_u8(self.optimal_bits);
Expand Down Expand Up @@ -331,17 +332,17 @@ impl FastPFOR {
/// unpacks regular values per block, patches in exceptions by position.
///
/// # Arguments
/// * `thissize` - Expected decompressed integer count
/// * `this_size` - Expected decompressed integer count
/// * `input_offset` - Advanced by bytes read
/// * `output_offset` - Advanced by `thissize`
/// * `output_offset` - Advanced by `this_size`
#[expect(clippy::too_many_lines)]
fn decode_page(
&mut self,
input: &[u32],
input_offset: &mut Cursor<u32>,
output: &mut [u32],
output_offset: &mut Cursor<u32>,
thissize: u32,
this_size: u32,
) -> FastPForResult<()> {
let n = u32::try_from(input.len())
.map_err(|_| FastPForError::InvalidInputLength(input.len()))?;
Expand All @@ -362,7 +363,7 @@ impl FastPFOR {
// The C++ encoder uses a raw `memcpy` of bytes into the u32 output (no endian
// conversion), and the decoder does a raw reinterpret_cast back -- both native byte
// order. `cast_slice` is the exact Rust equivalent: a safe, zero-copy native view.
let input_bytes: &[u8] = bytemuck::cast_slice(input);
let input_bytes: &[u8] = cast_slice(input);
let mut byte_pos = (inexcept as usize)
.checked_mul(4)
.filter(|&bp| bp <= input_bytes.len())
Expand Down Expand Up @@ -448,7 +449,7 @@ impl FastPFOR {
let mut tmp_output_offset = output_offset.position() as u32;
let mut tmp_input_offset = input_offset.position() as u32;

let run_end = thissize / self.block_size;
let run_end = this_size / self.block_size;
for _ in 0..run_end {
let bits = input_bytes.get_val(byte_pos)?;
if bits > 32 {
Expand Down
Loading