Move numpy _import_array out of static initializers into Python module initialization

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Adam Glustein <Adam.Glustein@Point72.com>

Author: AdamGlustein

cpp/csp/python/CMakeLists.txt

@@ -119,6 +119,12 @@ target_link_libraries(cspnpstatsimpl cspimpl npstatsimpl)
 target_include_directories(npstatsimpl PRIVATE ${NUMPY_INCLUDE_DIRS})
 target_include_directories(cspnpstatsimpl PRIVATE ${NUMPY_INCLUDE_DIRS})
 
+# By default numpy declares PyArray_API as a TU-local `static` — each translation unit gets its own copy and must call import_array() itself
+# Setting PY_ARRAY_UNIQUE_SYMBOL turns it into a named extern symbol, so all TUs within one .so share a single pointer and a single import_array() from PyInit_* is enough
+target_compile_definitions(cspimpl PRIVATE PY_ARRAY_UNIQUE_SYMBOL=CSP_CSPIMPL_ARRAY_API)
+target_compile_definitions(npstatsimpl PRIVATE PY_ARRAY_UNIQUE_SYMBOL=CSP_CSPNPSTATSIMPL_ARRAY_API)
+target_compile_definitions(cspnpstatsimpl PRIVATE PY_ARRAY_UNIQUE_SYMBOL=CSP_CSPNPSTATSIMPL_ARRAY_API)
+
 install(TARGETS csptypesimpl cspimpl cspbaselibimpl cspbasketlibimpl cspmathimpl cspstatsimpl csptestlibimpl cspnpstatsimpl
         PUBLIC_HEADER DESTINATION include/csp/python
         RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR}

cpp/csp/python/NumpyConversions.cpp

@@ -1,4 +1,4 @@
-//this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyConversions.h
+#define NO_IMPORT_ARRAY
 #include <numpy/ndarrayobject.h>
 #include <numpy/npy_2_compat.h>
 
@@ -9,14 +9,6 @@
 #include <locale>
 #include <codecvt>
 
-static void * init_nparray()
-{
-    csp::python::AcquireGIL gil;
-    import_array();
-    return nullptr;
-}
-static void * s_init_array = init_nparray();
-
 namespace csp::python
 {
 

cpp/csp/python/PyNumpyAdapter.cpp

@@ -1,30 +1,17 @@
-//this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyInputAdapter.h
+#define NO_IMPORT_ARRAY
 #include <numpy/ndarrayobject.h>
-//
-// BUT!  really we can't have this here and in NumpyConversions.cpp....
-//
 
 #include <csp/python/Conversions.h>
 #include <csp/python/Exception.h>
 #include <csp/python/NumpyInputAdapter.h>
 #include <csp/python/PyEngine.h>
 #include <csp/python/PyInputAdapterWrapper.h>
 
-static void * init_nparray()
-{
-    csp::python::AcquireGIL gil;
-    import_array();
-    return nullptr;
-}
-static void * s_init_array;
-
 namespace csp::python
 {
 
 static InputAdapter * numpy_adapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args )
 {
-    s_init_array = init_nparray();
-
     PyObject * type;
     PyArrayObject * pyDatetimes = nullptr;
     PyArrayObject * pyValues    = nullptr;

cpp/csp/python/adapters/ArrowCppNodes.cpp

@@ -4,7 +4,7 @@
 // RecordBatches are transported across the Python/C++ boundary as PyCapsules
 // using the Arrow C Data Interface.
 
-// Must include numpy first without NO_IMPORT_ARRAY to define PyArray_API in this TU
+#define NO_IMPORT_ARRAY
 #include <numpy/ndarrayobject.h>
 #include <csp/adapters/arrow/RecordBatchToStruct.h>
 #include <csp/adapters/arrow/StructToRecordBatch.h>
@@ -268,11 +268,8 @@ EXPORT_CPPNODE( struct_to_record_batches );
 REGISTER_CPPNODE( csp::cppnodes, record_batches_to_struct );
 REGISTER_CPPNODE( csp::cppnodes, struct_to_record_batches );
 
-// import_array() expands to `return NULL` on error, so the enclosing function must return a pointer type
 static void * _init_numpy = []() -> void *
 {
-    csp::python::AcquireGIL gil;
-    import_array();
     csp::python::registerNumpyListFieldReaderFactory();
     csp::python::registerNumpyListFieldWriterFactory();
     return nullptr;

cpp/csp/python/adapters/CMakeLists.txt

@@ -2,6 +2,7 @@ if(CSP_BUILD_ARROW_ADAPTER)
     add_library(arrowadapterimpl SHARED PyArrowInputAdapter.cpp ArrowCppNodes.cpp ArrowInputAdapter.h)
     target_link_libraries(arrowadapterimpl csp_core csp_engine cspimpl csp_arrow_adapter ${CSP_ARROW_LINK_LIBS})
     target_include_directories(arrowadapterimpl PUBLIC ${ARROW_INCLUDE_DIR})
+    target_compile_definitions(arrowadapterimpl PRIVATE PY_ARRAY_UNIQUE_SYMBOL=CSP_ARROWADAPTERIMPL_ARRAY_API)
     install(TARGETS arrowadapterimpl RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} )
 endif()
 

cpp/csp/python/adapters/PyArrowInputAdapter.cpp

@@ -1,3 +1,5 @@
+#include <numpy/ndarrayobject.h>
+
 #include <csp/python/adapters/ArrowInputAdapter.h>
 #include <csp/python/Conversions.h>
 #include <csp/python/Exception.h>
@@ -49,6 +51,9 @@ PyMODINIT_FUNC PyInit__arrowadapterimpl( void )
         return NULL;
     }
 
+    // Call _import_array from PyInit rather than a static initializer to avoid running the numpy import machinery at dlopen time, which could deadlock
+    import_array();
+
     if( !InitHelper::instance().execute( m ) )
     {
         return NULL;

cpp/csp/python/cspbaselibimpl.cpp

@@ -6,14 +6,6 @@
 #include <csp/engine/CppNode.h>
 #include <csp/python/Conversions.h>
 
-static void * init_nparray()
-{
-    csp::python::AcquireGIL gil;
-    import_array();
-    return nullptr;
-}
-static void * s_init_array = init_nparray();
-
 namespace csp::cppnodes
 {
 DECLARE_CPPNODE( exprtk_impl )
@@ -377,6 +369,9 @@ PyMODINIT_FUNC PyInit__cspbaselibimpl(void)
     if( m == NULL )
         return NULL;
 
+    // Call _import_array from PyInit rather than a static initializer to avoid running the numpy import machinery at dlopen time, which could deadlock
+    import_array();
+
     if( !csp::python::InitHelper::instance().execute( m ) )
         return NULL;
 

cpp/csp/python/cspimpl.cpp

@@ -1,3 +1,5 @@
+#include <numpy/ndarrayobject.h>
+
 #include <csp/engine/DynamicEngine.h>
 #include <csp/python/Conversions.h>
 #include <csp/python/InitHelper.h>
@@ -156,6 +158,9 @@ PyMODINIT_FUNC PyInit__cspimpl(void)
     if( m == NULL )
         return NULL;
 
+    // Call _import_array from PyInit rather than a static initializer to avoid running the numpy import machinery at dlopen time, which could deadlock
+    import_array();
+
     if( !InitHelper::instance().execute( m ) )
         return NULL;
 

cpp/csp/python/cspnpstatsimpl.cpp

@@ -71,6 +71,9 @@ PyMODINIT_FUNC PyInit__cspnpstatsimpl(void)
     if( m == NULL )
         return NULL;
 
+    // Call _import_array from PyInit rather than a static initializer to avoid running the numpy import machinery at dlopen time, which could deadlock
+    import_array();
+
     if( !csp::python::InitHelper::instance().execute( m ) )
         return NULL;
 

cpp/csp/python/npstatsimpl.cpp

@@ -1,6 +1,8 @@
 
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 #include <Python.h> // needs to be included first
+
+#define NO_IMPORT_ARRAY
 #include <numpy/ndarrayobject.h>
 #include <numpy/npy_math.h> // need to be included before csp
 
@@ -26,14 +28,6 @@ template<> struct NPY_TYPE<double>   { static const int value = NPY_DOUBLE; };
 
 using namespace csp::cppnodes;
 
-static void * init_nparray()
-{
-    csp::python::AcquireGIL gil;
-    import_array();
-    return nullptr;
-}
-static void * s_init_array = init_nparray();
-
 // NumPy specific statistic functions
 
 namespace csp::python