Add flag for controling how string should the casing between int and float be

Add option to have default values for aggregations

Make the default value type a variant

Author: vasil-pashov

cpp/arcticdb/async/tasks.hpp

@@ -14,9 +14,7 @@
 #include <arcticdb/storage/storage.hpp>
 #include <arcticdb/storage/key_segment_pair.hpp>
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/hash.hpp>
 #include <arcticdb/stream/stream_utils.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/stream/stream_source.hpp>
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/variant_key.hpp>
@@ -30,7 +28,6 @@
 #include <arcticdb/util/test/random_throw.hpp>
 
 #include <type_traits>
-#include <ranges>
 
 namespace arcticdb::async {
 

cpp/arcticdb/column_store/column.cpp

@@ -371,9 +371,9 @@ void Column::mark_absent_rows(size_t num_rows) {
     }
 }
 
-void Column::default_initialize_rows(size_t start_pos, size_t num_rows, bool ensure_alloc) {
+void Column::default_initialize_rows(size_t start_pos, size_t num_rows, bool ensure_alloc, const VariantRawValue default_value) {
     if (num_rows > 0) {
-        type_.visit_tag([this, start_pos, num_rows, ensure_alloc](auto tag) {
+        type_.visit_tag([&,this](auto tag) {
             using T = std::decay_t<decltype(tag)>;
             using RawType = typename T::DataTypeTag::raw_type;
             const auto bytes = (num_rows * sizeof(RawType));
@@ -382,7 +382,12 @@ void Column::default_initialize_rows(size_t start_pos, size_t num_rows, bool ens
                 data_.ensure<uint8_t>(bytes);
 
             auto type_ptr = data_.ptr_cast<RawType>(start_pos, bytes);
-            util::default_initialize<T>(reinterpret_cast<uint8_t *>(type_ptr), bytes);
+            if (std::holds_alternative<RawType>(default_value)) {
+                const RawType raw_default = std::get<RawType>(default_value);
+                std::fill_n(type_ptr, num_rows, raw_default);
+            } else {
+                util::default_initialize<T>(reinterpret_cast<uint8_t *>(type_ptr), bytes);
+            }
 
             if (ensure_alloc)
                 data_.commit();

cpp/arcticdb/column_store/column.hpp

@@ -20,12 +20,10 @@
 #include <arcticdb/util/preconditions.hpp>
 #include <arcticdb/util/sparse_utils.hpp>
 
-#include <folly/container/Enumerate.h>
 // Compilation fails on Mac if cstdio is not included prior to folly/Function.h due to a missing definition of memalign in folly/Memory.h
 #ifdef __APPLE__
 #include <cstdio>
 #endif
-#include <folly/Function.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 
@@ -453,7 +451,7 @@ class Column {
 
     void mark_absent_rows(size_t num_rows);
 
-    void default_initialize_rows(size_t start_pos, size_t num_rows, bool ensure_alloc);
+    void default_initialize_rows(size_t start_pos, size_t num_rows, bool ensure_alloc, VariantRawValue default_value = {});
 
     void set_row_data(size_t row_id);
 
@@ -730,19 +728,15 @@ class Column {
         size_t end_row
     );
 
-    template <
-            typename input_tdt,
-            typename functor>
+    template <typename input_tdt, typename functor>
     requires std::is_invocable_r_v<void, functor, typename input_tdt::DataTypeTag::raw_type>
     static void for_each(const Column& input_column, functor&& f) {
         auto input_data = input_column.data();
         std::for_each(input_data.cbegin<input_tdt>(), input_data.cend<input_tdt>(), std::forward<functor>(f));
     }
 
-    template <
-            typename input_tdt,
-            typename functor>
-    requires std::is_invocable_r_v<void, functor, typename ColumnData::Enumeration<typename input_tdt::DataTypeTag::raw_type>>
+    template <typename input_tdt, typename functor>
+    requires std::is_invocable_r_v<void, functor, ColumnData::Enumeration<typename input_tdt::DataTypeTag::raw_type>>
     static void for_each_enumerated(const Column& input_column, functor&& f) {
         auto input_data = input_column.data();
         if (input_column.is_sparse()) {
@@ -754,10 +748,7 @@ class Column {
         }
     }
 
-    template <
-            typename input_tdt,
-            typename output_tdt,
-            typename functor>
+    template <typename input_tdt, typename output_tdt, typename functor>
     requires std::is_invocable_r_v<typename output_tdt::DataTypeTag::raw_type, functor, typename input_tdt::DataTypeTag::raw_type>
     static void transform(const Column& input_column, Column& output_column, functor&& f) {
         auto input_data = input_column.data();
@@ -839,9 +830,7 @@ class Column {
         }
     }
 
-    template <
-            typename input_tdt,
-            std::predicate<typename input_tdt::DataTypeTag::raw_type> functor>
+    template <typename input_tdt, std::predicate<typename input_tdt::DataTypeTag::raw_type> functor>
     static void transform(const Column& input_column,
                           util::BitSet& output_bitset,
                           bool sparse_missing_value_output,

cpp/arcticdb/column_store/memory_segment_impl.cpp

@@ -13,6 +13,8 @@
 
 #include <google/protobuf/any.pb.h>
 
+#include <arcticdb/stream/index.hpp>
+
 namespace arcticdb {
 
     SegmentInMemoryImpl::SegmentInMemoryImpl() = default;
@@ -154,7 +156,7 @@ std::optional<std::size_t> SegmentInMemoryImpl::column_index(std::string_view na
     if (auto index = column_index(name); index)
         return index;
 
-    std::string multi_index_column_name = fmt::format("__idx__{}", name);
+    const std::string multi_index_column_name = stream::mangled_name(name);
     if (auto multi_index = column_index(multi_index_column_name); multi_index)
         return multi_index;
 

cpp/arcticdb/column_store/string_pool.hpp

@@ -7,8 +7,6 @@
 
 #pragma once
 
-#include <cstddef>
-#include <cstdint>
 #include <string_view>
 #include <unordered_set>
 

cpp/arcticdb/entity/stream_descriptor.hpp

@@ -11,7 +11,6 @@
 #include "arcticdb/storage/memory_layout.hpp"
 
 #include <arcticdb/entity/field_collection_proto.hpp>
-#include <arcticdb/util/variant.hpp>
 #include <arcticdb/entity/types_proto.hpp>
 
 #include <ankerl/unordered_dense.h>
@@ -308,13 +307,27 @@ struct OutputSchema {
         column_types().emplace(name, data_type);
     }
 
+    auto release() {
+        column_types_.reset();
+        return std::tuple{std::move(stream_descriptor_), std::move(norm_metadata_), std::move(default_values_)};
+    }
+
+    void clear_default_values() {
+        default_values_.clear();
+    }
+
+    void set_default_value_for_column(const std::string_view name, const VariantRawValue& value) {
+        default_values_.emplace(std::string(name), value);
+    }
+
 private:
     StreamDescriptor stream_descriptor_;
     std::optional<ankerl::unordered_dense::map<std::string, DataType>> column_types_;
+    ankerl::unordered_dense::map<std::string, VariantRawValue> default_values_;
 };
 
 template <class IndexType>
-inline void set_index(StreamDescriptor &stream_desc) {
+void set_index(StreamDescriptor &stream_desc) {
     stream_desc.set_index_field_count(std::uint32_t(IndexType::field_count()));
     stream_desc.set_index_type(IndexType::type());
 }

cpp/arcticdb/entity/type_utils.cpp

@@ -67,9 +67,30 @@ namespace arcticdb {
         return std::make_optional<entity::TypeDescriptor>(combine_data_type(entity::ValueType::FLOAT, target_size), dimension);
     }
 
+    static bool is_valid_int_to_float_conversion(
+        const entity::TypeDescriptor& source,
+        const entity::TypeDescriptor& target,
+        IntToFloatConversion int_to_to_float_conversion
+    ) {
+        debug::check<ErrorCode::E_ASSERTION_FAILURE>(
+            entity::is_integer_type(source.data_type()) && is_floating_point_type(target.data_type()),
+            "Expected source to be int and target to be float got: {} {}", source, target
+        );
+        if (int_to_to_float_conversion == IntToFloatConversion::STRICT) {
+            return target.get_size_bits() == entity::SizeBits::S64 || source.get_size_bits() < entity::SizeBits::S32;
+        } else {
+            debug::check<ErrorCode::E_ASSERTION_FAILURE>(
+                int_to_to_float_conversion == IntToFloatConversion::PERMISSIVE,
+                "Unknown int to float conversion option: {}", static_cast<int>(int_to_to_float_conversion)
+            );
+            return true;
+        }
+    }
+
     bool is_valid_type_promotion_to_target(
         const entity::TypeDescriptor& source,
-        const entity::TypeDescriptor& target
+        const entity::TypeDescriptor& target,
+        IntToFloatConversion int_to_to_float_conversion
     ) {
         if (source.dimension() != target.dimension()) {
             // Empty of dimension 0 means lack of any given type and can be promoted to anything (even if the dimensions
@@ -108,7 +129,7 @@ namespace arcticdb {
                 return target_size > source_size;
             } else if (is_floating_point_type(target_type)) {
                 // UINT->FLOAT
-                return target_size == entity::SizeBits::S64 || source_size < entity::SizeBits::S32;
+                return is_valid_int_to_float_conversion(source, target, int_to_to_float_conversion);
             } else {
                 // Non-numeric target type
                 return false;
@@ -122,7 +143,7 @@ namespace arcticdb {
                 return target_size >= source_size;
             } else if (is_floating_point_type(target_type)) {
                 // INT->FLOAT
-                return target_size == entity::SizeBits::S64 || source_size < entity::SizeBits::S32;
+                return is_valid_int_to_float_conversion(source, target, int_to_to_float_conversion);
             } else {
                 // Non-numeric target type
                 return false;
@@ -175,10 +196,14 @@ namespace arcticdb {
         }
     }
 
-    std::optional<entity::TypeDescriptor> has_valid_common_type(const entity::TypeDescriptor& left, const entity::TypeDescriptor& right) {
-        if (is_valid_type_promotion_to_target(left, right)) {
+    std::optional<entity::TypeDescriptor> has_valid_common_type(
+        const entity::TypeDescriptor& left,
+        const entity::TypeDescriptor& right,
+        IntToFloatConversion int_to_to_float_conversion
+    ) {
+        if (is_valid_type_promotion_to_target(left, right, int_to_to_float_conversion)) {
             return right;
-        } else if (is_valid_type_promotion_to_target(right, left)) {
+        } else if (is_valid_type_promotion_to_target(right, left, int_to_to_float_conversion)) {
             return left;
         }
 

cpp/arcticdb/entity/type_utils.hpp

@@ -8,7 +8,6 @@
 #pragma once
 #include<optional>
 #include <fmt/format.h>
-#include<arcticdb/entity/descriptors.hpp>
 #include <arcticdb/entity/types.hpp>
 
 namespace arcticdb {
@@ -17,18 +16,30 @@ namespace entity {
     struct TypeDescriptor;
 }
 
+/// Defines which static casts from int to float are permitted in is_valid_type_promotion_to_target
+enum class IntToFloatConversion {
+    /// Avoids lossy casting from int to float and tries to make sure that the integer can be represented exactly using
+    /// the specified float, (u)int8, (u)int16 can be represented exactly using float 32, (u)int32 can be represented
+    /// exactly via float64. Note this still allows casting (u)int64 to flaot64 even though it's a lossy cast.
+    STRICT,
+    /// Allow all type casts from int to float regardless of the byte size of the int and float type
+    PERMISSIVE
+};
+
 /// Two types are trivially compatible if their byte representation is exactly the same i.e. you can memcpy
 /// n elements of left type from one buffer to n elements of type right in another buffer and get the same result
 [[nodiscard]] bool trivially_compatible_types(const entity::TypeDescriptor& left, const entity::TypeDescriptor& right);
 
 [[nodiscard]] bool is_valid_type_promotion_to_target(
     const entity::TypeDescriptor& source,
-    const entity::TypeDescriptor& target
+    const entity::TypeDescriptor& target,
+    IntToFloatConversion int_to_to_float_conversion = IntToFloatConversion::STRICT
 );
 
 [[nodiscard]] std::optional<entity::TypeDescriptor> has_valid_common_type(
     const entity::TypeDescriptor& left,
-    const entity::TypeDescriptor& right
+    const entity::TypeDescriptor& right,
+    IntToFloatConversion int_to_to_float_conversion = IntToFloatConversion::STRICT
 );
 
 inline std::string get_user_friendly_type_string(const entity::TypeDescriptor& type) {

cpp/arcticdb/entity/types.hpp

@@ -394,6 +394,20 @@ DATA_TYPE_TAG(EMPTYVAL, std::uint64_t)
 DATA_TYPE_TAG(BOOL_OBJECT8, uint8_t)
 #undef DATA_TYPE_TAG
 
+using VariantRawValue = std::variant<
+    std::monostate,
+    int8_t,
+    int16_t,
+    int32_t,
+    int64_t,
+    uint8_t,
+    uint16_t,
+    uint32_t,
+    uint64_t,
+    bool,
+    float,
+    double>;
+
 enum class Dimension : uint8_t {
     Dim0 = 0,
     Dim1 = 1,

cpp/arcticdb/pipeline/frame_slice_map.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include <arcticdb/pipeline/pipeline_context.hpp>
+#include <arcticdb/pipeline/index_utils.hpp>
 
 namespace arcticdb::pipelines {
 

cpp/arcticdb/pipeline/frame_utils.cpp

@@ -8,6 +8,7 @@
 #include <arcticdb/pipeline/frame_utils.hpp>
 #include <arcticdb/stream/aggregator.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
+#include <arcticdb/entity/protobuf_mappings.hpp>
 
 namespace arcticdb {
 

cpp/arcticdb/pipeline/frame_utils.hpp

@@ -8,13 +8,11 @@
 #pragma once
 
 #include <arcticdb/pipeline/pipeline_context.hpp>
-#include <arcticdb/column_store/string_pool.hpp>
 #include <arcticdb/column_store/chunked_buffer.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
 #include <arcticdb/entity/atom_key.hpp>
 #include <arcticdb/pipeline/input_tensor_frame.hpp>
 #include <arcticdb/stream/protobuf_mappings.hpp>
-#include <arcticdb/entity/protobuf_mappings.hpp>
 #include <arcticdb/python/gil_lock.hpp>
 #include <arcticdb/python/python_types.hpp>
 #include <arcticdb/python/python_to_tensor_frame.hpp>
@@ -26,6 +24,10 @@
 
 namespace arcticdb {
 
+namespace pipelines::index {
+    struct IndexSegmentReader;
+}
+
 inline size_t get_first_string_size(const pipelines::PipelineContextRow& context_row, ChunkedBuffer &src, std::size_t first_row_in_frame) {
     auto offset = first_context_row(context_row.slice_and_key(), first_row_in_frame);
     auto num_rows = context_row.slice_and_key().slice_.row_range.diff();

cpp/arcticdb/pipeline/input_tensor_frame.hpp

@@ -12,18 +12,14 @@
 #include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/entity/index_range.hpp>
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/flatten_utils.hpp>
+#include <arcticdb/util/type_traits.hpp>
 
 namespace arcticdb::pipelines {
 
 using namespace arcticdb::entity;
 
-/// @TODO Move to a separate "util" header
-template <typename T, typename... U>
-concept is_any_of = (std::same_as<T, U> || ...);
-
 template <typename IndexT>
-concept ValidIndex = is_any_of<
+concept ValidIndex = util::any_of<
         std::remove_cvref_t<std::remove_pointer_t<std::decay_t<IndexT>>>,
         stream::TimeseriesIndex,
         stream::RowCountIndex,