chore: start refactoring to only use bytes by refactoring the ByteBuffer internals (#47)

this is the first of many PR, otherwise I cannot make this reviewable.

I am a abit tiread, will go over my changes tomorrow again

Works towards #35

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

Cargo.toml

@@ -23,9 +23,10 @@ harness = false
 # Keeping it on for now to simplify development.
 default = ["cpp"]
 cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"]
-rust = ["dep:thiserror"]
+rust = ["dep:thiserror", "dep:bytes"]
 
 [dependencies]
+bytes = { version = "1.11", optional = true }
 cxx = { version = "1.0.194", optional = true }
 thiserror = { version = "2.0.18", optional = true }
 

src/rust/bytebuffer.rs

@@ -2,7 +2,8 @@ use std::io::Cursor;
 
 use crate::rust::cursor::IncrementCursor;
 use crate::rust::integer_compression::{bitpacking, helpers};
-use crate::rust::{bytebuffer, FastPForResult, Integer, Skippable};
+use crate::rust::{FastPForResult, Integer, Skippable};
+use bytes::{Buf as _, BufMut as _, BytesMut};
 
 /// Block size constant for 256 integers per block
 pub const BLOCK_SIZE_256: u32 = 256;
@@ -29,7 +30,7 @@ pub struct FastPFOR {
     /// Exception values indexed by bit width difference
     pub data_to_be_packed: Vec<Vec<u32>>,
     /// Metadata buffer for encoding/decoding
-    pub bytes_container: bytebuffer::ByteBuffer,
+    pub bytes_container: BytesMut,
     /// Maximum integers per page
     pub page_size: u32,
     /// Position trackers for exception arrays
@@ -146,7 +147,9 @@ impl FastPFOR {
         FastPFOR {
             page_size,
             block_size,
-            bytes_container: bytebuffer::ByteBuffer::new(3 * page_size / block_size + page_size),
+            bytes_container: BytesMut::with_capacity(
+                (3 * page_size / block_size + page_size) as usize,
+            ),
             data_to_be_packed: vec![vec![0; page_size as usize / 32 * 4]; 33],
             data_pointers: vec![0; 33],
             freqs: vec![0; 33],
@@ -188,10 +191,10 @@ impl FastPFOR {
         while tmp_input_offset <= final_input_offset {
             self.best_b_from_data(input, tmp_input_offset);
             let tmp_best_b = self.optimal_bits;
-            self.bytes_container.put(self.optimal_bits as u8);
-            self.bytes_container.put(self.exception_count as u8);
+            self.bytes_container.put_u8(self.optimal_bits as u8);
+            self.bytes_container.put_u8(self.exception_count as u8);
             if self.exception_count > 0 {
-                self.bytes_container.put(self.max_bits as u8);
+                self.bytes_container.put_u8(self.max_bits as u8);
                 let index = self.max_bits - self.optimal_bits;
                 if self.data_pointers[index as usize] + self.exception_count as usize
                     >= self.data_to_be_packed[index as usize].len()
@@ -204,7 +207,7 @@ impl FastPFOR {
                 }
                 for k in 0..self.block_size {
                     if (input[(k + tmp_input_offset) as usize] >> self.optimal_bits) != 0 {
-                        self.bytes_container.put(k as u8);
+                        self.bytes_container.put_u8(k as u8);
                         self.data_to_be_packed[index as usize]
                             [self.data_pointers[index as usize]] =
                             input[(k + tmp_input_offset) as usize] >> tmp_best_b;
@@ -226,22 +229,23 @@ impl FastPFOR {
         }
         input_offset.set_position(u64::from(tmp_input_offset));
         output[header_pos] = tmp_output_offset - header_pos as u32;
-        let byte_size = self.bytes_container.position();
-        while (self.bytes_container.position() & 3) != 0 {
-            self.bytes_container.put(0);
+        let byte_size = self.bytes_container.len();
+        while (self.bytes_container.len() & 3) != 0 {
+            self.bytes_container.put_u8(0);
         }
         // Output should have 3 position as 4
-        output[tmp_output_offset as usize] = byte_size;
+        output[tmp_output_offset as usize] = byte_size as u32;
         tmp_output_offset += 1;
-        let how_many_ints = self.bytes_container.position() / 4;
-        self.bytes_container.flip();
-
-        self.bytes_container.as_int_buffer().get(
-            output,
-            tmp_output_offset as usize,
-            how_many_ints as usize,
-        );
-        tmp_output_offset += how_many_ints;
+        let how_many_ints = self.bytes_container.len() / 4;
+
+        for it in output
+            .iter_mut()
+            .skip(tmp_output_offset as usize)
+            .take(how_many_ints)
+        {
+            *it = self.bytes_container.get_u32_le();
+        }
+        tmp_output_offset += how_many_ints as u32;
         let mut bitmap = 0;
         for k in 2..=32 {
             if self.data_pointers[k] != 0 {
@@ -343,10 +347,9 @@ impl FastPFOR {
         inexcept += 1;
         self.bytes_container.clear();
         let length = bytesize.div_ceil(4);
-        self.bytes_container.buffer =
-            self.bytes_container
-                .as_int_buffer()
-                .put(input, inexcept as usize, length);
+        for i in inexcept..inexcept + length {
+            self.bytes_container.put_u32_le(input[i as usize]);
+        }
         inexcept += length;
 
         let bitmap = input[inexcept as usize];
@@ -405,8 +408,8 @@ impl FastPFOR {
 
         let run_end = thissize / self.block_size;
         for _ in 0..run_end {
-            let b = u32::from(self.bytes_container.get());
-            let cexcept = self.bytes_container.get();
+            let b = u32::from(self.bytes_container.get_u8());
+            let cexcept = self.bytes_container.get_u8();
             for k in (0..self.block_size).step_by(32) {
                 bitpacking::fast_unpack(
                     input,
@@ -418,16 +421,16 @@ impl FastPFOR {
                 tmp_input_offset += b;
             }
             if cexcept > 0 {
-                let maxbits = u32::from(self.bytes_container.get());
+                let maxbits = u32::from(self.bytes_container.get_u8());
                 let index = maxbits - b;
                 if index == 1 {
                     for _ in 0..cexcept {
-                        let pos = self.bytes_container.get();
+                        let pos = self.bytes_container.get_u8();
                         output[pos as usize + tmp_output_offset as usize] |= 1 << b;
                     }
                 } else {
                     for _ in 0..cexcept {
-                        let pos = self.bytes_container.get();
+                        let pos = self.bytes_container.get_u8();
                         let except_value = self.data_to_be_packed[index as usize]
                             [self.data_pointers[index as usize]];
                         output[pos as usize + tmp_output_offset as usize] |= except_value << b;

src/rust/integer_compression/fastpfor.rs

@@ -1,9 +1,9 @@
 use std::io::Cursor;
 
-use crate::rust::bytebuffer::ByteBuffer;
 use crate::rust::cursor::IncrementCursor;
 use crate::rust::integer_compression::helpers::{extract7bits, extract_7bits_maskless};
 use crate::rust::{FastPForError, FastPForResult, Integer, Skippable};
+use bytes::{Buf as _, BufMut as _, BytesMut};
 
 #[derive(Debug)]
 pub struct VariableByte;
@@ -35,43 +35,40 @@ impl Skippable for VariableByte {
             // Return early if there is no data to compress
             return Ok(());
         }
-        let mut buf = ByteBuffer::new(input_length * 8);
+        let mut buf = BytesMut::with_capacity(input_length as usize * 8);
         for k in input_offset.position()..(input_offset.position() + u64::from(input_length)) {
             let val = i64::from(input[k as usize]);
             if val < (1 << 7) {
-                buf.put((val | (1 << 7)) as u8);
+                buf.put_u8((val | (1 << 7)) as u8);
             } else if val < (1 << 14) {
-                buf.put(extract7bits(0, val));
-                buf.put(extract_7bits_maskless(1, val) | (1 << 7));
+                buf.put_u8(extract7bits(0, val));
+                buf.put_u8(extract_7bits_maskless(1, val) | (1 << 7));
             } else if val < (1 << 21) {
-                buf.put(extract7bits(0, val));
-                buf.put(extract7bits(1, val));
-                buf.put(extract_7bits_maskless(2, val) | (1 << 7));
+                buf.put_u8(extract7bits(0, val));
+                buf.put_u8(extract7bits(1, val));
+                buf.put_u8(extract_7bits_maskless(2, val) | (1 << 7));
             } else if val < (1 << 28) {
-                buf.put(extract7bits(0, val));
-                buf.put(extract7bits(1, val));
-                buf.put(extract7bits(2, val));
-                buf.put(extract_7bits_maskless(3, val) | (1 << 7));
+                buf.put_u8(extract7bits(0, val));
+                buf.put_u8(extract7bits(1, val));
+                buf.put_u8(extract7bits(2, val));
+                buf.put_u8(extract_7bits_maskless(3, val) | (1 << 7));
             } else {
-                buf.put(extract7bits(0, val));
-                buf.put(extract7bits(1, val));
-                buf.put(extract7bits(2, val));
-                buf.put(extract7bits(3, val));
-                buf.put(extract_7bits_maskless(4, val) | (1 << 7));
+                buf.put_u8(extract7bits(0, val));
+                buf.put_u8(extract7bits(1, val));
+                buf.put_u8(extract7bits(2, val));
+                buf.put_u8(extract7bits(3, val));
+                buf.put_u8(extract_7bits_maskless(4, val) | (1 << 7));
             }
         }
-        while buf.position() % 4 != 0 {
-            buf.put(0);
+        while buf.len() % 4 != 0 {
+            buf.put_u8(0);
         }
-        let length = buf.position();
-        buf.flip();
-        let ibuf = buf.as_int_buffer();
-        ibuf.get(
-            output,
-            output_offset.position() as usize,
-            (length / 4) as usize,
-        );
-        output_offset.add(length / 4);
+        let length = buf.len();
+        let output_position = output_offset.position() as usize;
+        for it in output.iter_mut().skip(output_position).take(length / 4) {
+            *it = buf.get_u32_le();
+        }
+        output_offset.add(length as u32 / 4);
         input_offset.add(input_length);
 
         Ok(())

src/rust/integer_compression/variable_byte.rs

@@ -1,4 +1,3 @@
-pub mod bytebuffer;
 mod cursor;
 mod error;
 mod integer_compression;