First pass to remove dependency on Python in arcticdb_core

Author: JohanMabille

cpp/arcticdb/CMakeLists.txt

@@ -194,7 +194,6 @@ set(arcticdb_srcs
         column_store/column_data.hpp
         column_store/column_data_random_accessor.hpp
         column_store/column.hpp
-        column_store/column_utils.hpp
         column_store/key_segment.hpp
         column_store/memory_segment.hpp
         column_store/memory_segment_impl.hpp
@@ -338,6 +337,7 @@ set(arcticdb_srcs
         util/buffer.hpp
         util/buffer_pool.hpp
         util/clock.hpp
+        util/concepts.hpp
         util/configs_map.hpp
         util/constants.hpp
         util/constructors.hpp

cpp/arcticdb/column_store/column.hpp

@@ -15,6 +15,7 @@
 #include <arcticdb/entity/performance_tracing.hpp>
 #include <arcticdb/entity/types.hpp>
 #include <arcticdb/util/bitset.hpp>
+#include <arcticdb/util/concepts.hpp>
 #include <arcticdb/util/cursored_buffer.hpp>
 #include <arcticdb/util/flatten_utils.hpp>
 #include <arcticdb/util/preconditions.hpp>
@@ -26,23 +27,14 @@
 #include <cstdio>
 #endif
 #include <folly/Function.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/numpy.h>
 
 #include <concepts>
 #include <numeric>
 #include <optional>
 
-namespace py = pybind11;
-
 namespace arcticdb {
 
-// this is needed to make templates of templates work
-// since py::array_t has more than one template parameter
-// (the rest are defaulted)
-template< class T>
-using py_array_t = py::array_t<T>;
-
+using util::arithmetic_tensor;
 using namespace arcticdb::entity;
 
 // N.B. this will not catch all the things that C++ considers to be narrowing conversions, because
@@ -383,10 +375,9 @@ class Column {
         return std::move(shapes_.buffer());
     }
 
-    template<class T, template<class> class Tensor, std::enable_if_t<
-            std::is_integral_v<T> || std::is_floating_point_v<T>,
-            int> = 0>
-    void set_array(ssize_t row_offset, Tensor<T> &val) {
+    template<arithmetic_tensor TensorType>
+    void set_array(ssize_t row_offset, TensorType& val) {
+        using value_type = typename TensorType::value_type;
         ARCTICDB_SAMPLE(ColumnSetArray, RMTSF_Aggregate)
         magic_.check();
         util::check_arg(last_logical_row_ + 1 == row_offset, "set_array expected row {}, actual {} ", last_logical_row_ + 1, row_offset);
@@ -395,26 +386,8 @@ class Column {
         memcpy(shapes_.cursor(), val.shape(), val.ndim() * sizeof(shape_t));
         auto info = val.request();
         util::FlattenHelper flatten(val);
-        auto data_ptr = reinterpret_cast<T*>(data_.cursor());
-        flatten.flatten(data_ptr, reinterpret_cast<const T *>(info.ptr));
-        update_offsets(val.nbytes());
-        data_.commit();
-        shapes_.commit();
-        ++last_logical_row_;
-    }
-
-    template<class T, std::enable_if_t< std::is_integral_v<T> || std::is_floating_point_v<T>, int> = 0>
-    void set_array(ssize_t row_offset, py::array_t<T>& val) {
-        ARCTICDB_SAMPLE(ColumnSetArray, RMTSF_Aggregate)
-        magic_.check();
-        util::check_arg(last_logical_row_ + 1 == row_offset, "set_array expected row {}, actual {} ", last_logical_row_ + 1, row_offset);
-        data_.ensure_bytes(val.nbytes());
-        shapes_.ensure<shape_t>(val.ndim());
-        memcpy(shapes_.cursor(), val.shape(), val.ndim() * sizeof(shape_t));
-        auto info = val.request();
-        util::FlattenHelper<T, py_array_t> flatten(val);
-        auto data_ptr = reinterpret_cast<T*>(data_.cursor());
-        flatten.flatten(data_ptr, reinterpret_cast<const T*>(info.ptr));
+        auto data_ptr = reinterpret_cast<value_type*>(data_.cursor());
+        flatten.flatten(data_ptr, reinterpret_cast<const value_type*>(info.ptr));
         update_offsets(val.nbytes());
         data_.commit();
         shapes_.commit();

cpp/arcticdb/column_store/column_utils.hpp

@@ -126,15 +126,8 @@ class SegmentInMemory {
         impl_->init_column_map();
     }
 
-    template<class T, template<class> class Tensor>
-    requires std::integral<T> || std::floating_point<T>
-    void set_array(position_t pos, Tensor<T> &val) {
-        impl_->set_array(pos, val);
-    }
-
-    template<class T>
-    requires std::integral<T> || std::floating_point<T>
-    void set_array(position_t pos, py::array_t<T>& val) {
+    template<arithmetic_tensor TensorType>
+    void set_array(position_t pos, TensorType &val) {
         impl_->set_array(pos, val);
     }
 

cpp/arcticdb/column_store/memory_segment.hpp

@@ -9,6 +9,7 @@
 
 #include <arcticdb/entity/types.hpp>
 #include <arcticdb/column_store/column.hpp>
+#include <arcticdb/util/concepts.hpp>
 #include <arcticdb/util/offset_string.hpp>
 #include <arcticdb/util/preconditions.hpp>
 
@@ -506,17 +507,8 @@ class SegmentInMemoryImpl {
         set_string(idx, val);
     }
 
-    template<class T, template<class> class Tensor>
-    requires std::integral<T> || std::floating_point<T>
-    void set_array(position_t pos, Tensor<T> &val) {
-        magic_.check();
-        ARCTICDB_SAMPLE(MemorySegmentSetArray, 0)
-        column_unchecked(pos).set_array(row_id_ + 1, val);
-    }
-
-    template<class T>
-    requires std::integral<T> || std::floating_point<T>
-    void set_array(position_t pos, py::array_t<T>& val) {
+    template<arithmetic_tensor TensorType>
+    void set_array(position_t pos, TensorType &val) {
         magic_.check();
         ARCTICDB_SAMPLE(MemorySegmentSetArray, 0)
         column_unchecked(pos).set_array(row_id_ + 1, val);

cpp/arcticdb/column_store/memory_segment_impl.hpp

@@ -28,7 +28,15 @@ void register_column_store(py::module &m) {
     py::class_<StringPool>(m, "StringPool")
         .def(py::init())
         .def_property_readonly("nbytes", &StringPool::size)
-        .def("as_buffer_info", &StringPool::as_buffer_info);
+        .def("as_buffer_info", [](const StringPool& s) {
+                return py::buffer_info{
+                    (void *) s.get_const_view(0).data(),
+                    1,
+                    py::format_descriptor<char>::format(),
+                    ssize_t(s.get_const_view(0).size())
+                    
+                };
+            });
 }
 
 } // namespace arcticc::column_store

cpp/arcticdb/column_store/python_bindings.cpp

@@ -10,8 +10,6 @@
 #include <arcticdb/column_store/segment_utils.hpp>
 #include <ankerl/unordered_dense.h>
 
-#include <pybind11/pybind11.h>
-
 namespace arcticdb {
 
 /*****************
@@ -199,15 +197,6 @@ size_t StringPool::size() const {
     return block_.size();
 }
 
-py::buffer_info StringPool::as_buffer_info() const {
-    return py::buffer_info{
-        (void *) block_.at(0).data(),
-        1,
-        py::format_descriptor<char>::format(),
-        ssize_t(block_.at(0).size())
-    };
-}
-
 std::optional<position_t> StringPool::get_offset_for_column(std::string_view string, const Column& column) {
     auto unique_values = unique_values_for_string_column(column);
     remove_nones_and_nans(unique_values);

cpp/arcticdb/column_store/string_pool.cpp

@@ -18,12 +18,6 @@
 #include <arcticdb/column_store/chunked_buffer.hpp>
 #include <arcticdb/column_store/column_data.hpp>
 
-namespace pybind11 {
-    struct buffer_info;
-}
-
-namespace py = pybind11;
-
 #include <ankerl/unordered_dense.h>
 
 namespace arcticdb {
@@ -172,8 +166,6 @@ class StringPool {
 
     [[nodiscard]] size_t num_blocks() const;
 
-    py::buffer_info as_buffer_info() const;
-
     std::optional<position_t> get_offset_for_column(std::string_view str, const Column& column);
     ankerl::unordered_dense::set<position_t> get_offsets_for_column(const std::shared_ptr<std::unordered_set<std::string>>& strings, const Column& column);
   private:

cpp/arcticdb/column_store/string_pool.hpp

@@ -14,10 +14,6 @@
 // for std::accumulate
 #include <numeric>
 
-#include <pybind11/numpy.h>
-
-namespace py = pybind11;
-
 namespace arcticdb::entity {
 
 inline ssize_t calc_elements(const shape_t* shape, ssize_t ndim) {
@@ -166,6 +162,7 @@ ssize_t byte_offset_impl(const stride_t* strides, ssize_t i, Ix... index) {
 //TODO is the conversion to a typed tensor really necessary for the codec part?
 template<typename T>
 struct TypedTensor : public NativeTensor {
+    using value_type = T;
     static size_t itemsize() { return sizeof(T); }
 
     std::array<stride_t, 2> f_style_strides() {
@@ -255,12 +252,8 @@ struct TypedTensor : public NativeTensor {
         }  
     }
 };
-template<typename T>
-py::array to_py_array(const TypedTensor<T>& tensor) {
-    return py::array({tensor.shape(), tensor.shape() + tensor.ndim()}, reinterpret_cast<const T*>(tensor.data()));
-}
 
 template<typename T>
 using TensorType = TypedTensor<T>;
 
-}//namespace arcticdb
+}//namespace arcticdb