Remove vendored pyarrow code and upgrade arrow to 19 (#507)

* Remove vendored pyarrow code

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

* Pin cmake version<3.27 to enable vcpkg build

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

* Upgrade vcpkg, and pin arrow to 19.0.1 in vcpkg.json

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

* Upgrade arrow pin to <20 in conda, pyproject.toml

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

* Address comments

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

---------

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>

Author: arhamchopra

conda/dev-environment-unix.yml

@@ -16,7 +16,7 @@ dependencies:
  - graphviz
  - gtest
  - httpx>=0.20,<1
- - libarrow=16
+ - libarrow<20
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
  - librdkafka
@@ -31,7 +31,7 @@ dependencies:
  - pillow
  - polars
  - psutil
- - pyarrow>=15,<19
+ - pyarrow>=15,<20
  - pydantic>=2
  - pytest
  - pytest-asyncio

conda/dev-environment-win.yml

@@ -16,7 +16,7 @@ dependencies:
  - graphviz
  - gtest
  - httpx>=0.20,<1
- - libarrow=16
+ - libarrow<20
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
  - librdkafka
@@ -31,7 +31,7 @@ dependencies:
  - pillow
  - polars
  - psutil
- - pyarrow>=7,<19
+ - pyarrow>=15,<20
  - pydantic>=2
  - pytest
  - pytest-asyncio

cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp

@@ -21,10 +21,9 @@ void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std:
     ::parquet::ArrowWriterProperties::Builder arrowBuilder;
     arrowBuilder.store_schema();
 
-    STATUS_OK_OR_THROW_RUNTIME(
-        ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build(),
-                                            &m_fileWriter ),
-            "Failed to open parquet file writer" );
+    auto res = ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build() );
+    STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to open parquet file writer" );
+    m_fileWriter = res.MoveValueUnsafe();
 }
 
 void ParquetFileWriterWrapper::close()

cpp/csp/python/adapters/CMakeLists.txt

@@ -6,36 +6,9 @@ if(CSP_BUILD_KAFKA_ADAPTER)
 endif()
 
 if(CSP_BUILD_PARQUET_ADAPTER)
-    set(VENDORED_PYARROW_ROOT "${CMAKE_SOURCE_DIR}/cpp/csp/python/adapters/vendored/pyarrow-16.0.0/")
-    set(ARROW_PYTHON_SRCS
-        ${VENDORED_PYARROW_ROOT}/arrow/python/arrow_to_pandas.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/benchmark.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/common.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/datetime.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/decimal.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/deserialize.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/extension_type.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/gdb.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/helpers.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/inference.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/init.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/io.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/ipc.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_convert.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_to_arrow.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/python_to_arrow.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/pyarrow.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/serialize.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
-        ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc)
-    add_library(parquetadapterimpl SHARED parquetadapterimpl.cpp ${ARROW_PYTHON_SRCS})
-    # Ignore warning regarding static datetime API initialization coming from vendored Arrow code
-    if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-        set_target_properties(parquetadapterimpl PROPERTIES COMPILE_OPTIONS "-Wno-unused-variable")
-    endif()
+    add_library(parquetadapterimpl SHARED parquetadapterimpl.cpp)
     target_link_libraries(parquetadapterimpl csp_core csp_engine cspimpl csp_parquet_adapter)
-    target_include_directories(parquetadapterimpl PUBLIC ${ARROW_INCLUDE_DIR} ${PARQUET_INCLUDE_DIR} "${VENDORED_PYARROW_ROOT}")
-    target_compile_definitions(parquetadapterimpl PUBLIC ARROW_PYTHON_STATIC)
+    target_include_directories(parquetadapterimpl PUBLIC ${ARROW_INCLUDE_DIR} ${PARQUET_INCLUDE_DIR})
     install(TARGETS parquetadapterimpl RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} )
 endif()
 

cpp/csp/python/adapters/parquetadapterimpl.cpp

@@ -17,7 +17,6 @@
 #include <csp/python/PyCppNode.h>
 #include <csp/python/PyNodeWrapper.h>
 #include <csp/python/NumpyConversions.h>
-#include <arrow/python/pyarrow.h>
 #include <arrow/io/memory.h>
 #include <arrow/ipc/reader.h>
 #include <locale>
@@ -77,21 +76,6 @@ REGISTER_CPPNODE( csp::cppnodes, parquet_dict_basket_writer );
 namespace
 {
 
-struct PyArrowInitializer
-{
-    static inline bool ensureInitialized()
-    {
-        static bool initialized = false;
-        if( unlikely( !initialized ) )
-        {
-            if( arrow::py::import_pyarrow() != 0 )
-                CSP_THROW( csp::python::PythonPassthrough, "" );
-            initialized = true;
-        }
-        return initialized;
-    }
-};
-
 class FileNameGenerator : public csp::Generator<std::string, csp::DateTime, csp::DateTime>
 {
 public:
@@ -145,7 +129,6 @@ class ArrowTableGenerator : public csp::Generator<std::shared_ptr<arrow::Table>,
     ArrowTableGenerator( PyObject *wrappedGenerator )
             : m_wrappedGenerator( csp::python::PyObjectPtr::incref( wrappedGenerator ) )
     {
-        PyArrowInitializer::ensureInitialized();
     }
 
     void init( csp::DateTime start, csp::DateTime end ) override