feat: teach ALPRDArray how to append_to_builder

The PrimitiveBuilder API is a bit awkward for this. I needed two key things:

1. I want to treat a trailing slice of the builder's values as a different type. In particular,
   ALPRD decodes u32s (respectively u64s) and then converts them to f32s (resp. f64s).

2. I want to separately update the nulls and the values. In particular, ALPRD stores the nulls on
   the left parts whereas the right parts hold the values.

All of this is copacetic *when done on the UninitRange*, but I do it on the whole
PrimitiveBuilder. I kinda think PrimitiveBuilder should be able to split itself into the initialized
and uninitialized parts. In my opinion, it seems reasonable to transmute the builder of
uninitialized parts since no one else is looking at that data anyway.

Signed-off-by: Daniel King <dan@spiraldb.com>

Author: danking

encodings/alp/src/alp_rd/array.rs

@@ -19,6 +19,8 @@ use vortex_array::ProstMetadata;
 use vortex_array::SerializeMetadata;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::buffer::BufferHandle;
+use vortex_array::builders::ArrayBuilder;
+use vortex_array::builders::PrimitiveBuilder;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
 use vortex_array::dtype::PType;
@@ -36,13 +38,16 @@ use vortex_array::vtable::ValidityChild;
 use vortex_array::vtable::ValidityVTableFromChild;
 use vortex_array::vtable::VisitorVTable;
 use vortex_buffer::Buffer;
+use vortex_error::VortexExpect as _;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_mask::Mask;
 use vortex_session::VortexSession;
 
+use crate::ALPRDFloat;
+use crate::alp_rd::alp_rd_decode_inplace;
 use crate::alp_rd::kernel::PARENT_KERNELS;
 use crate::alp_rd::rules::RULES;
 use crate::alp_rd_decode;
@@ -222,6 +227,18 @@ impl VTable for ALPRDVTable {
         Ok(())
     }
 
+    fn append_to_builder(
+        array: &Self::Array,
+        builder: &mut dyn ArrayBuilder,
+        ctx: &mut ExecutionCtx,
+    ) -> VortexResult<()> {
+        if array.is_f32() {
+            append_to_builder_typed::<f32>(array, builder, ctx)
+        } else {
+            append_to_builder_typed::<f64>(array, builder, ctx)
+        }
+    }
+
     fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
         let left_parts = array.left_parts().clone().execute::<PrimitiveArray>(ctx)?;
         let right_parts = array.right_parts().clone().execute::<PrimitiveArray>(ctx)?;
@@ -282,6 +299,58 @@ impl VTable for ALPRDVTable {
     }
 }
 
+fn append_to_builder_typed<T: ALPRDFloat>(
+    array: &ALPRDArray,
+    builder: &mut dyn ArrayBuilder,
+    ctx: &mut ExecutionCtx,
+) -> VortexResult<()> {
+    let left_parts = array.left_parts().clone().execute::<PrimitiveArray>(ctx)?;
+    let validity = array
+        .left_parts()
+        .validity()?
+        .to_array(array.len())
+        .execute::<Mask>(ctx)?;
+
+    let builder: &mut PrimitiveBuilder<T> = builder
+        .as_any_mut()
+        .downcast_mut()
+        .vortex_expect("ALPRD array must canonicalize into a primitive array");
+
+    let cursor = builder.len();
+    let n = array.len();
+    builder.reserve_exact(n);
+
+    builder.as_values_and_nulls(|non_nullable_builder, nulls| -> VortexResult<()> {
+        // SAFETY: T and T::UINT have the same size/alignment (f32↔u32, f64↔u64).
+        unsafe {
+            non_nullable_builder.as_transmuted::<T::UINT, VortexResult<()>>(|uint_builder| {
+                // BitPacked can downcast uint_builder to PrimitiveBuilder<T::UINT>.
+                array.right_parts().append_to_builder(uint_builder, ctx)?;
+
+                // Decode in-place: combine left+right bits.
+                let right_parts_slice = &mut uint_builder.values_mut()[cursor..cursor + n];
+                alp_rd_decode_inplace::<T>(
+                    left_parts.into_buffer::<u16>(),
+                    array.left_parts_dictionary(),
+                    array.right_bit_width,
+                    right_parts_slice,
+                    array.left_parts_patches(),
+                    ctx,
+                )?;
+
+                Ok(())
+            })?;
+        }
+
+        // Set the correct validity from left_parts.
+        nulls.append_validity_mask(validity);
+
+        Ok(())
+    })?;
+
+    Ok(())
+}
+
 #[derive(Clone, Debug)]
 pub struct ALPRDArray {
     dtype: DType,

encodings/alp/src/alp_rd/mod.rs

@@ -294,13 +294,46 @@ pub fn alp_rd_decode<T: ALPRDFloat>(
     left_parts: Buffer<u16>,
     left_parts_dict: &[u16],
     right_bit_width: u8,
-    right_parts: BufferMut<T::UINT>,
+    mut right_parts: BufferMut<T::UINT>,
     left_parts_patches: Option<&Patches>,
     ctx: &mut ExecutionCtx,
 ) -> VortexResult<Buffer<T>> {
-    if left_parts.len() != right_parts.len() {
-        vortex_panic!("alp_rd_decode: left_parts.len != right_parts.len");
-    }
+    alp_rd_decode_inplace::<T>(
+        left_parts,
+        left_parts_dict,
+        right_bit_width,
+        right_parts.as_mut_slice(),
+        left_parts_patches,
+        ctx,
+    )?;
+
+    // SAFETY: alp_rd_decode_inplace combined left+right bits in-place.
+    // The buffer now contains valid T bit patterns stored as T::UINT.
+    Ok(unsafe { right_parts.transmute::<T>() }.freeze())
+}
+
+/// Decode ALP-RD encoded values in-place into a mutable slice of right parts.
+///
+/// After this function returns, `right_parts` contains valid `T` bit patterns
+/// (stored as `T::UINT`). Each element is the combination of the dictionary-decoded
+/// left part shifted left by `right_bit_width` bits, OR'd with the original right part.
+///
+/// # Panics
+///
+/// Panics if `left_parts.len() != right_parts.len()`.
+pub fn alp_rd_decode_inplace<T: ALPRDFloat>(
+    left_parts: Buffer<u16>,
+    left_parts_dict: &[u16],
+    right_bit_width: u8,
+    right_parts: &mut [T::UINT],
+    left_parts_patches: Option<&Patches>,
+    ctx: &mut ExecutionCtx,
+) -> VortexResult<()> {
+    assert_eq!(
+        left_parts.len(),
+        right_parts.len(),
+        "alp_rd_decode_inplace: left_parts.len != right_parts.len"
+    );
 
     // Decode the left-parts dictionary
     let mut values = BufferMut::<u16>::from_iter(
@@ -316,13 +349,13 @@ pub fn alp_rd_decode<T: ALPRDFloat>(
         alp_rd_apply_patches(&mut values, &indices, &patch_values, patches.offset());
     }
 
-    // Shift the left-parts and add in the right-parts.
-    Ok(alp_rd_decode_core(
-        left_parts_dict,
-        right_bit_width,
-        right_parts,
-        values,
-    ))
+    // Combine left + right in-place: *right = (left << rbw) | *right
+    for (right, left) in right_parts.iter_mut().zip(values.iter()) {
+        let left_uint = <T as ALPRDFloat>::from_u16(*left);
+        *right = (left_uint << (right_bit_width as usize)) | *right;
+    }
+
+    Ok(())
 }
 
 /// Apply patches to the decoded left-parts values.
@@ -343,25 +376,6 @@ fn alp_rd_apply_patches(
     })
 }
 
-/// Core decode logic shared between `alp_rd_decode` and `execute_alp_rd_decode`.
-fn alp_rd_decode_core<T: ALPRDFloat>(
-    _left_parts_dict: &[u16],
-    right_bit_width: u8,
-    right_parts: BufferMut<T::UINT>,
-    values: BufferMut<u16>,
-) -> Buffer<T> {
-    // Shift the left-parts and add in the right-parts.
-    let mut index = 0;
-    right_parts
-        .map_each_in_place(|right| {
-            let left = values[index];
-            index += 1;
-            let left = <T as ALPRDFloat>::from_u16(left);
-            T::from_bits((left << (right_bit_width as usize)) | right)
-        })
-        .freeze()
-}
-
 /// Find the best "cut point" for a set of floating point values such that we can
 /// cast them all to the relevant value instead.
 fn find_best_dictionary<T: ALPRDFloat>(samples: &[T]) -> ALPRDDictionary {

vortex-array/src/builders/lazy_null_builder.rs

@@ -31,6 +31,10 @@ impl LazyBitBufferBuilder {
         }
     }
 
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+
     /// Appends `n` non-null values to the builder.
     #[inline]
     pub fn append_n_non_nulls(&mut self, n: usize) {
@@ -105,6 +109,11 @@ impl LazyBitBufferBuilder {
         self.inner.as_ref().map(|i| i.len()).unwrap_or(self.len)
     }
 
+    /// Returns true if this builder contains no bits.
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
     fn finish(&mut self) -> Option<BitBuffer> {
         self.len = 0;
         self.inner.take().map(|b| b.freeze())

vortex-array/src/builders/mod.rs

@@ -44,7 +44,7 @@ use crate::match_each_native_ptype;
 use crate::scalar::Scalar;
 
 mod lazy_null_builder;
-pub(crate) use lazy_null_builder::LazyBitBufferBuilder;
+pub use lazy_null_builder::LazyBitBufferBuilder;
 
 mod bool;
 mod decimal;

vortex-array/src/builders/primitive.rs

@@ -118,6 +118,97 @@ impl<T: NativePType> PrimitiveBuilder<T> {
         self.values.extend(iter);
         self.nulls.append_validity_mask(mask);
     }
+
+    /// Returns a mutable slice of the initialized values in this builder.
+    pub fn values_mut(&mut self) -> &mut [T] {
+        self.values.as_mut_slice()
+    }
+
+    /// Temporarily detaches the nulls buffer, making this builder non-nullable
+    /// for the duration of the closure.
+    ///
+    /// The closure receives:
+    /// - `&mut self` with dtype changed to non-nullable and nulls swapped out
+    /// - The detached [`LazyBitBufferBuilder`] for managing validity separately
+    ///
+    /// After the closure returns, the original nullability and nulls are restored.
+    pub fn as_values_and_nulls<R>(
+        &mut self,
+        f: impl FnOnce(&mut Self, &mut LazyBitBufferBuilder) -> R,
+    ) -> R {
+        let original_dtype = self.dtype.clone();
+
+        // Make the builder non-nullable.
+        self.dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable);
+
+        // Swap nulls out.
+        let mut nulls = LazyBitBufferBuilder::new(0);
+        std::mem::swap(&mut self.nulls, &mut nulls);
+
+        let result = f(self, &mut nulls);
+
+        // Restore dtype and nulls.
+        self.dtype = original_dtype;
+        std::mem::swap(&mut self.nulls, &mut nulls);
+
+        result
+    }
+
+    /// Temporarily transmutes the values buffer from `T` to `U` and runs a
+    /// closure with a `PrimitiveBuilder<U>`.
+    ///
+    /// The temporary builder preserves this builder's nullability, and the nulls
+    /// buffer is moved into it for the duration of the closure.
+    ///
+    /// After the closure returns, values are transmuted back to `T` and the
+    /// nulls buffer is restored.
+    ///
+    /// # Safety
+    ///
+    /// - `T` and `U` must have the same size and alignment.
+    /// - After this call, any newly appended values contain `U` bit patterns.
+    ///   The caller must ensure these are valid `T` bit patterns before reading
+    ///   them as `T`.
+    pub unsafe fn as_transmuted<U: NativePType, R>(
+        &mut self,
+        f: impl FnOnce(&mut PrimitiveBuilder<U>) -> R,
+    ) -> R {
+        assert_eq!(
+            size_of::<T>(),
+            size_of::<U>(),
+            "T and U must have the same size"
+        );
+        assert_eq!(
+            align_of::<T>(),
+            align_of::<U>(),
+            "T and U must have the same alignment"
+        );
+
+        let values = std::mem::take(&mut self.values);
+        // SAFETY: We have verified that T and U have the same size and alignment.
+        let retyped_values: BufferMut<U> = unsafe { values.transmute() };
+
+        // Swap nulls into the temp builder so it inherits our validity state.
+        let mut nulls = LazyBitBufferBuilder::new(0);
+        std::mem::swap(&mut self.nulls, &mut nulls);
+
+        let mut temp_builder = PrimitiveBuilder {
+            dtype: DType::Primitive(U::PTYPE, self.dtype.nullability()),
+            values: retyped_values,
+            nulls,
+        };
+
+        let result = f(&mut temp_builder);
+
+        // Restore values (transmute back to T).
+        // SAFETY: Same size/alignment guarantee as above.
+        self.values = unsafe { std::mem::take(&mut temp_builder.values).transmute() };
+
+        // Restore nulls.
+        std::mem::swap(&mut self.nulls, &mut temp_builder.nulls);
+
+        result
+    }
 }
 
 impl<T: NativePType> ArrayBuilder for PrimitiveBuilder<T> {