[DO NOT MERGE] Experimental array scalar value

vortex-array/src/arrays/fixed_size_list/vtable/operations.rs

@@ -6,22 +6,14 @@ use vortex_error::VortexResult;
 use crate::arrays::FixedSizeListArray;
 use crate::arrays::FixedSizeListVTable;
 use crate::scalar::Scalar;
+use crate::scalar::ScalarValue;
 use crate::vtable::OperationsVTable;
 
 impl OperationsVTable<FixedSizeListVTable> for FixedSizeListVTable {
     fn scalar_at(array: &FixedSizeListArray, index: usize) -> VortexResult<Scalar> {
         // By the preconditions we know that the list scalar is not null.
         let list = array.fixed_size_list_elements_at(index)?;
-        let children_elements: Vec<Scalar> = (0..list.len())
-            .map(|i| list.scalar_at(i))
-            .collect::<VortexResult<_>>()?;
-
-        debug_assert_eq!(children_elements.len(), array.list_size() as usize);
-
-        Ok(Scalar::fixed_size_list(
-            list.dtype().clone(),
-            children_elements,
-            array.dtype().nullability(),
-        ))
+        let scalar_value = ScalarValue::Array(list);
+        Scalar::try_new(array.dtype().clone(), Some(scalar_value))
     }
 }

vortex-array/src/arrays/list/vtable/operations.rs

@@ -1,27 +1,19 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use std::sync::Arc;
-
 use vortex_error::VortexResult;
 
 use crate::arrays::ListArray;
 use crate::arrays::ListVTable;
 use crate::scalar::Scalar;
+use crate::scalar::ScalarValue;
 use crate::vtable::OperationsVTable;
 
 impl OperationsVTable<ListVTable> for ListVTable {
     fn scalar_at(array: &ListArray, index: usize) -> VortexResult<Scalar> {
         // By the preconditions we know that the list scalar is not null.
-        let elems = array.list_elements_at(index)?;
-        let scalars: Vec<Scalar> = (0..elems.len())
-            .map(|i| elems.scalar_at(i))
-            .collect::<VortexResult<_>>()?;
-
-        Ok(Scalar::list(
-            Arc::new(elems.dtype().clone()),
-            scalars,
-            array.dtype().nullability(),
-        ))
+        let list = array.list_elements_at(index)?;
+        let scalar_value = ScalarValue::Array(list);
+        Scalar::try_new(array.dtype().clone(), Some(scalar_value))
     }
 }

vortex-array/src/arrays/listview/vtable/operations.rs

@@ -1,27 +1,19 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use std::sync::Arc;
-
 use vortex_error::VortexResult;
 
 use crate::arrays::ListViewArray;
 use crate::arrays::ListViewVTable;
 use crate::scalar::Scalar;
+use crate::scalar::ScalarValue;
 use crate::vtable::OperationsVTable;
 
 impl OperationsVTable<ListViewVTable> for ListViewVTable {
     fn scalar_at(array: &ListViewArray, index: usize) -> VortexResult<Scalar> {
         // By the preconditions we know that the list scalar is not null.
         let list = array.list_elements_at(index)?;
-        let children: Vec<Scalar> = (0..list.len())
-            .map(|i| list.scalar_at(i))
-            .collect::<VortexResult<_>>()?;
-
-        Ok(Scalar::list(
-            Arc::new(list.dtype().clone()),
-            children,
-            array.dtype.nullability(),
-        ))
+        let scalar_value = ScalarValue::Array(list);
+        Scalar::try_new(array.dtype().clone(), Some(scalar_value))
     }
 }

vortex-array/src/scalar/proto.rs

@@ -110,6 +110,14 @@ impl From<&ScalarValue> for pb::ScalarValue {
                     kind: Some(Kind::ListValue(ListValue { values })),
                 }
             }
+            ScalarValue::Array(_) => {
+                // We simply expand the value out into a list to serialize it correctly.
+                let expanded = value.expand_array().vortex_expect(
+                    "something went wrong when expanding scalar value for serialization",
+                );
+
+                Self::from(&expanded)
+            }
         }
     }
 }

vortex-array/src/scalar/scalar_value.rs

@@ -4,14 +4,19 @@
 //! Core [`ScalarValue`] type definition.
 
 use std::cmp::Ordering;
-use std::fmt::Display;
-use std::fmt::Formatter;
+use std::fmt;
+use std::hash::Hash;
+use std::hash::Hasher;
 
 use itertools::Itertools;
 use vortex_buffer::BufferString;
 use vortex_buffer::ByteBuffer;
+use vortex_error::VortexExpect;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
 use vortex_error::vortex_panic;
 
+use crate::ArrayRef;
 use crate::dtype::DType;
 use crate::scalar::DecimalValue;
 use crate::scalar::PValue;
@@ -20,7 +25,7 @@ use crate::scalar::PValue;
 ///
 /// This enum represents the possible non-null values that can be stored in a scalar. When the
 /// scalar is null, the value is represented as `None` in the `Option<ScalarValue>` field.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone)]
 pub enum ScalarValue {
     /// A boolean value.
     Bool(bool),
@@ -34,6 +39,8 @@ pub enum ScalarValue {
     Binary(ByteBuffer),
     /// A list of potentially null scalar values.
     List(Vec<Option<ScalarValue>>),
+    /// An Array. This is NOT serializable, and we ONLY use this for performance reasons.
+    Array(ArrayRef),
 }
 
 impl ScalarValue {
@@ -102,25 +109,25 @@ impl ScalarValue {
             }
         })
     }
-}
 
-impl PartialOrd for ScalarValue {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        match (self, other) {
-            (ScalarValue::Bool(a), ScalarValue::Bool(b)) => a.partial_cmp(b),
-            (ScalarValue::Primitive(a), ScalarValue::Primitive(b)) => a.partial_cmp(b),
-            (ScalarValue::Decimal(a), ScalarValue::Decimal(b)) => a.partial_cmp(b),
-            (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => a.partial_cmp(b),
-            (ScalarValue::Binary(a), ScalarValue::Binary(b)) => a.partial_cmp(b),
-            (ScalarValue::List(a), ScalarValue::List(b)) => a.partial_cmp(b),
-            // (ScalarValue::Extension(a), ScalarValue::Extension(b)) => a.partial_cmp(b),
-            _ => None,
-        }
+    /// Turns a scalar value `Array` variant into a `List` variant.
+    ///
+    /// TODO
+    pub fn expand_array(&self) -> VortexResult<Self> {
+        let Self::Array(array) = self else {
+            vortex_bail!("tried to expand a scalar value that was not an array");
+        };
+
+        let values = (0..array.len())
+            .map(|i| array.scalar_at(i).map(|scalar| scalar.value().cloned()))
+            .collect::<VortexResult<Vec<_>>>()?;
+
+        Ok(Self::List(values))
     }
 }
 
-impl Display for ScalarValue {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for ScalarValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             ScalarValue::Bool(b) => write!(f, "{b}"),
             ScalarValue::Primitive(p) => write!(f, "{p}"),
@@ -163,6 +170,83 @@ impl Display for ScalarValue {
                 }
                 write!(f, "]")
             }
+            ScalarValue::Array(_) => {
+                // We simply expand the value out into a list to display it.
+                let expanded = self.expand_array().vortex_expect(
+                    "something went wrong when expanding scalar value for displaying",
+                );
+
+                expanded.fmt(f)
+            }
+        }
+    }
+}
+
+impl PartialEq for ScalarValue {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (ScalarValue::Bool(a), ScalarValue::Bool(b)) => a == b,
+            (ScalarValue::Primitive(a), ScalarValue::Primitive(b)) => a == b,
+            (ScalarValue::Decimal(a), ScalarValue::Decimal(b)) => a == b,
+            (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => a == b,
+            (ScalarValue::Binary(a), ScalarValue::Binary(b)) => a == b,
+            (ScalarValue::List(a), ScalarValue::List(b)) => a == b,
+            (ScalarValue::Array(_), ScalarValue::Array(_)) => {
+                // We simply expand the value out into a list before doing any comparison.
+                let lhs = self.expand_array().vortex_expect(
+                    "something went wrong when expanding lhs scalar value for comparison",
+                );
+                let rhs = other.expand_array().vortex_expect(
+                    "something went wrong when expanding rhs scalar value for comparison",
+                );
+
+                lhs == rhs
+            }
+            _ => false,
+        }
+    }
+}
+
+impl Eq for ScalarValue {}
+
+impl PartialOrd for ScalarValue {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match (self, other) {
+            (ScalarValue::Bool(a), ScalarValue::Bool(b)) => a.partial_cmp(b),
+            (ScalarValue::Primitive(a), ScalarValue::Primitive(b)) => a.partial_cmp(b),
+            (ScalarValue::Decimal(a), ScalarValue::Decimal(b)) => a.partial_cmp(b),
+            (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => a.partial_cmp(b),
+            (ScalarValue::Binary(a), ScalarValue::Binary(b)) => a.partial_cmp(b),
+            (ScalarValue::List(a), ScalarValue::List(b)) => a.partial_cmp(b),
+            (ScalarValue::Array(_), ScalarValue::Array(_)) => {
+                // We simply expand the value out into a list before doing any comparison.
+                let lhs = self.expand_array().vortex_expect(
+                    "something went wrong when expanding lhs scalar value for comparison",
+                );
+                let rhs = other.expand_array().vortex_expect(
+                    "something went wrong when expanding rhs scalar value for comparison",
+                );
+
+                lhs.partial_cmp(&rhs)
+            }
+            _ => None,
+        }
+    }
+}
+
+impl Hash for ScalarValue {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            ScalarValue::Bool(b) => b.hash(state),
+            ScalarValue::Primitive(p) => p.hash(state),
+            ScalarValue::Decimal(d) => d.hash(state),
+            ScalarValue::Utf8(s) => s.hash(state),
+            ScalarValue::Binary(b) => b.hash(state),
+            ScalarValue::List(l) => l.hash(state),
+            ScalarValue::Array(_) => self
+                .expand_array()
+                .vortex_expect("something went wrong when expanding scalar value for hashing")
+                .hash(state),
         }
     }
 }

vortex-array/src/scalar/validate.rs

@@ -74,6 +74,13 @@ impl Scalar {
                 );
             }
             DType::List(elem_dtype, _) => {
+                if let ScalarValue::Array(array) = value {
+                    // If the scalar value is an array, we just need to check that the array
+                    // elements match the elem_dtype, so just check that the dtype is the same.
+                    vortex_ensure_eq!(elem_dtype.as_ref(), array.dtype());
+                    return Ok(());
+                }
+
                 let ScalarValue::List(elements) = value else {
                     vortex_bail!("list dtype expected List value, got {value}");
                 };
@@ -84,6 +91,13 @@ impl Scalar {
                 }
             }
             DType::FixedSizeList(elem_dtype, size, _) => {
+                if let ScalarValue::Array(array) = value {
+                    // If the scalar value is an array, we just need to check that the array
+                    // elements match the elem_dtype, so just check that the dtype is the same.
+                    vortex_ensure_eq!(elem_dtype.as_ref(), array.dtype());
+                    return Ok(());
+                }
+
                 let ScalarValue::List(elements) = value else {
                     vortex_bail!("fixed-size list dtype expected List value, got {value}",);
                 };