folly cython bindings OSS build fixes

.gitignore

@@ -34,3 +34,8 @@ folly/FingerprintTables.cpp
 _build
 # Ignore all files generated by Buck2
 buck-out/
+
+.vscode
+.venv
+.python-version
+__pycache__

CMakeLists.txt

@@ -185,16 +185,18 @@ endif()
 
 list(REMOVE_ITEM files
   ${FOLLY_DIR}/python/error.cpp
-  ${FOLLY_DIR}/python/executor.cpp
+  ${FOLLY_DIR}/python/executor_intf.cpp
   ${FOLLY_DIR}/python/fibers.cpp
   ${FOLLY_DIR}/python/GILAwareManualExecutor.cpp
-  ${FOLLY_DIR}/python/iobuf.cpp
+  ${FOLLY_DIR}/python/iobuf_intf.cpp
   ${FOLLY_DIR}/python/iobuf_ext.cpp
   ${FOLLY_DIR}/python/ProactorExecutor.cpp
 )
 list(REMOVE_ITEM hfiles
   ${FOLLY_DIR}/python/fibers.h
   ${FOLLY_DIR}/python/GILAwareManualExecutor.h
+  ${FOLLY_DIR}/python/executor_intf.h
+  ${FOLLY_DIR}/python/iobuf_intf.h
   ${FOLLY_DIR}/python/iobuf_ext.h
   ${FOLLY_DIR}/python/ProactorExecutor.h
 )

build/fbcode_builder/manifests/libiberty

@@ -22,5 +22,8 @@ patchfile = libiberty_install_pic_lib.patch
 
 # only build the parts needed for demangling
 # as we still want to use system linker and assembler etc
+# XXX flags below should be taken from cmd args - getdeps.py change required XXX
 [autoconf.args]
 --enable-install-libiberty
+CFLAGS = -fPIC
+CXXFLAGS = -fPIC

build/fbcode_builder/manifests/ninja

@@ -24,9 +24,9 @@ builder = nop
 ninja.exe = bin/ninja.exe
 
 [download.not(os=windows)]
-url = https://github.com/ninja-build/ninja/archive/v1.10.2.tar.gz
-sha256 = ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed
+url = https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz
+sha256 = 821bdff48a3f683bc4bb3b6f0b5fe7b2d647cf65d52aeb63328c91a6c6df285a
 
 [build.not(os=windows)]
 builder = ninja_bootstrap
-subdir = ninja-1.10.2
+subdir = ninja-1.12.1

build/fbcode_builder/manifests/xz

@@ -12,8 +12,10 @@ xz
 xz-devel
 
 [download]
-url = https://tukaani.org/xz/xz-5.2.5.tar.gz
-sha256 = f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10
+url = https://github.com/tukaani-project/xz/archive/refs/tags/v5.2.5.tar.gz
+sha256 = 0d2b89629f13dd1a0602810529327195eff5f62a0142ccd65b903bc16a4ac78a
+# url = https://tukaani.org/xz/xz-5.2.5.tar.gz
+# sha256 = f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10
 
 [build]
 builder = autoconf

folly/CMakeLists.txt

@@ -83,10 +83,11 @@ if (PYTHON_EXTENSIONS)
       CC="${CMAKE_C_COMPILER}" CXX="${CMAKE_CXX_COMPILER}"
       python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py
       build_ext -f ${incs} ${libs}
-    BYPRODUCTS
-      ${_cybld}/folly/executor_api.h
-      ${_cybld}/folly/iobuf_api.h
-    WORKING_DIRECTORY ${_cybld}
+      BYPRODUCTS 
+        ${_cybld}/folly/executor_api.h
+        ${_cybld}/folly/iobuf_api.h
+        ${_cybld}/folly/fiber_manager_api.h
+      WORKING_DIRECTORY ${_cybld}
   )
 
   add_custom_target(create_post_binding_symlink ALL)
@@ -160,4 +161,8 @@ if (PYTHON_EXTENSIONS)
       WORKING_DIRECTORY ${_cybld}
     )"
   )
+  if(result)
+    message(FATAL_ERROR \"Setup.py failed with error code \${result}:\n\${error}\")
+  endif()
+")
 endif ()

folly/python/AsyncioExecutor.h

@@ -76,8 +76,10 @@ class AsyncioExecutor : public DrivableExecutor {
 
  private:
   static bool isPyFinalizing() noexcept {
-#if PY_VERSION_HEX <= 0x03070000
+#if PY_VERSION_HEX <= 0x03070000     // unsupported anyway - can remove
     return false;
+#elif PY_VERSION_HEX >= 0x030D0000  // Python 3.13+
+    return Py_IsFinalizing();
 #else
     return _Py_IsFinalizing();
 #endif

folly/python/__init__.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from libcpp cimport bool as cbool
 
 cdef extern from "folly/ExceptionWrapper.h" namespace "folly":
@@ -47,7 +48,7 @@ cdef extern from "folly/Unit.h" namespace "folly":
 cdef extern from "folly/futures/Promise.h" namespace "folly":
     cdef cppclass cFollyPromise "folly::Promise"[T]:
         cFollyPromise()
-        cFollyPromise(cFollyPromise&&)
+        cFollyPromise(cFollyPromise) # && removed
         void setValue[M](M& value)
         void setException[E](E& value)
         cFollyFuture[T] getFuture()

folly/python/async_generator.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from folly.coro cimport cFollyCoroTask
 
 cdef extern from "folly/python/async_generator.h" namespace "folly::coro":

folly/python/build_mode.pyx

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from folly cimport cFollyIsDebug, cFollyIsTsan
 
 folly_is_debug_build = bool(cFollyIsDebug)

folly/python/cast.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 cdef extern from "folly/lang/Cast.h" namespace "folly" nogil:
     cdef T down_cast_ref "folly::down_cast"[T, S](S ref)
     cdef T* down_cast_ptr "folly::down_cast"[T, S](S* ptr)

folly/python/coro.h

@@ -28,7 +28,7 @@
 #include <folly/CancellationToken.h>
 #include <folly/coro/Task.h>
 #include <folly/python/AsyncioExecutor.h>
-#include <folly/python/executor.h>
+#include <folly/python/executor_intf.h>
 
 #if FOLLY_HAS_COROUTINES
 

folly/python/coro.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from cpython.ref cimport PyObject
 from folly cimport cFollyExecutor, cFollyTry
 from libcpp cimport bool
@@ -32,20 +33,20 @@ cdef extern from "folly/CancellationToken.h" namespace "folly" nogil:
 cdef extern from "folly/python/coro.h" namespace "folly::python":
     void bridgeCoroTask[T](
         cFollyCoroTask[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )
     # No clue but cython overloading is getting confused so we alias
     void bridgeCoroTaskWith "folly::python::bridgeCoroTask"[T](
         cFollyExecutor* executor,
         cFollyCoroTask[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )
     void bridgeCoroTaskWithCancellation "folly::python::bridgeCoroTask"[T](
         cFollyExecutor* executor,
         cFollyCoroTask[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture,
         cFollyCancellationToken&& cancelToken,
     )

folly/python/error.cpp

@@ -30,7 +30,7 @@ namespace {
 // Best effort c-api implementation of repr(obj)
 std::string pyObjectToString(PyObject* obj) {
   constexpr StringPiece kConversionFail = "Error conversion failed";
-  PyObject *pyStr, *pyBytes;
+  PyObject *pyStr = nullptr, *pyBytes = nullptr;
   SCOPE_EXIT {
     Py_XDECREF(pyStr);
     Py_XDECREF(pyBytes);
@@ -44,16 +44,13 @@ std::string pyObjectToString(PyObject* obj) {
   }
 
   char* cStr = nullptr;
-#if PY_VERSION_HEX < 0x0300000
-  cStr = PyString_AsString(pyStr);
-#else
+
   pyBytes = PyUnicode_AsEncodedString(pyStr, "utf-8", "strict");
   if (pyBytes == nullptr) {
     return std::string(kConversionFail);
   }
 
   cStr = PyBytes_AsString(pyBytes);
-#endif
 
   if (cStr == nullptr) {
     return std::string(kConversionFail);
@@ -65,7 +62,7 @@ std::string pyObjectToString(PyObject* obj) {
 } // namespace
 
 void handlePythonError(StringPiece errPrefix) {
-  PyObject *ptype, *pvalue, *ptraceback;
+  PyObject *ptype = nullptr, *pvalue = nullptr, *ptraceback = nullptr;
   SCOPE_EXIT {
     Py_XDECREF(ptype);
     Py_XDECREF(pvalue);

folly/python/executor.pxd

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# distutils: language = c++
 
 from libcpp.memory cimport unique_ptr
 from folly cimport cFollyExecutor
@@ -48,7 +47,6 @@ cdef class AsyncioExecutor:
 cdef class ProactorExecutor(AsyncioExecutor):
     cdef unique_ptr[cProactorExecutor, cProactorExecutorDeleter] cQ
 
-
 cdef class IocpQueue(dict):
     cdef ProactorExecutor _executor
 

folly/python/executor.pyx

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 import asyncio
 import sys
 

folly/python/executor.cpp → folly/python/executor_intf.cpp

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-#include <folly/python/executor.h>
+#include <folly/python/executor_intf.h>
 
 #include <stdexcept>
 
-#include <folly/python/executor_api.h> // @manual
+#include <folly/executor_api.h> // @manual
 #include <folly/python/import.h>
 
 namespace folly {

folly/python/expected.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 cdef extern from "folly/Expected.h" namespace "folly" nogil:
     cdef cppclass cExpected "folly::Expected"[T, E]:
         cExpected(T&& val)

folly/python/fbstring.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from libcpp.string cimport string
 
 cdef extern from "folly/FBString.h" namespace "folly":

folly/python/fiber_manager.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from libcpp.memory cimport unique_ptr
 from libcpp cimport bool
 from folly.executor cimport cAsyncioExecutor

folly/python/fiber_manager.pyx

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 import asyncio
+from cpython cimport PyObject
 from cython.operator cimport dereference as deref
 from cpython.weakref cimport PyWeakref_NewRef, PyWeakref_GetObject
-from cpython cimport PyObject
+
 from libcpp.memory cimport unique_ptr
 from folly.executor cimport get_executor
 from libcpp.cast cimport (
@@ -73,9 +75,10 @@ cdef cFiberManager* get_fiber_manager(const cFiberManagerOptions& opts):
 
     loop = asyncio.get_event_loop()
     cdef FiberManager manager = None
-
+    # deprecated: PyWeakref_GetObject - migrate to PyWeakref_GetRef
     if last_loop is not None and <PyObject*>loop is PyWeakref_GetObject(last_loop):
         manager = last_manager
+
     if manager is None:
         try:
             manager = <FiberManager>(loop_to_controller[loop])

folly/python/fibers.cpp

@@ -18,7 +18,7 @@
 
 #include <stdexcept>
 
-#include <folly/python/fiber_manager_api.h>
+#include <folly/fiber_manager_api.h>
 #include <folly/python/import.h>
 
 namespace folly {

folly/python/fibers.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from cpython.ref cimport PyObject
 from folly cimport cFollyTry
 

folly/python/function.pxd

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 cdef extern from "folly/Function.h" namespace "folly" nogil:
     cdef cppclass cFunction "folly::Function"[T]:
         cFunction()

folly/python/futures.h

@@ -25,7 +25,7 @@
 #include <folly/Executor.h>
 #include <folly/futures/Future.h>
 #include <folly/python/AsyncioExecutor.h>
-#include <folly/python/executor.h>
+#include <folly/python/executor_intf.h>
 
 namespace folly {
 namespace python {

folly/python/futures.pxd

@@ -12,31 +12,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from cpython.ref cimport PyObject
 from folly cimport cFollyTry, cFollyFuture, cFollyExecutor, cFollySemiFuture
 
 cdef extern from "folly/python/futures.h" namespace "folly::python":
     void bridgeFuture[T](
         cFollyFuture[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )
     # No clue but cython overloading is getting confused so we alias
     void bridgeFutureWith "folly::python::bridgeFuture"[T](
         cFollyExecutor* executor,
         cFollyFuture[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )
     void bridgeSemiFuture[T](
         cFollySemiFuture[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )
     # No clue but cython overloading is getting confused so we alias
     void bridgeSemiFutureWith "folly::python::bridgeSemiFuture"[T](
         cFollyExecutor* executor,
         cFollySemiFuture[T]&& fut,
-        void(*)(cFollyTry[T]&&, PyObject*),
+        void(*)(cFollyTry[T]&&, PyObject*) noexcept,
         PyObject* pyFuture
     )