Skip to content

Commit 2341342

Browse files
AarjunCmeta-codesync[bot]
AarjunC
authored and
meta-codesync[bot]
committed
Fix OSS build and add thrift-python CMake module
Summary: Fix fbthrift-python OSS build (getdeps) on CentOS Stream 9 with Python 3.12, Cython 3.2.4, GCC 12 (gcc-toolset-12). Also add a generic CMake module (FBThriftPythonLibrary.cmake) for any project migrating from thrift-py-deprecated to thrift-python. Build fixes (xplat/thrift/lib/python/CMakeLists.txt + setup.py + manifests): 1. Add EVENT__LIBRARY_TYPE=STATIC to libevent-python manifest 2. Add lib64 search paths for folly .pxd files (CentOS/RHEL/Fedora) 3. unset(_folly_pxd_file CACHE) to prevent stale CMake cache 4. Add __PYX_ENUM_CLASS_DECL= define (cython/cython#5973) 5. Add CWD to Cython include path in setup.py CMake module (opensource/fbcode_builder/CMake/FBThriftPythonLibrary.cmake): Generic, project-agnostic module with three components: - fb_find_thrift_python_runtime(): discovers thrift-python and folly-python site-packages at configure time; auto-installs fbthrift-python wheel from getdeps build output if not pip-installed - add_fbthrift_python_library(): generates thrift_types.py via mstch_python generator; drops FBThrift::thrift_py from DEPENDS (C extensions resolved at runtime) - add_fb_thrift_python_executable(): drop-in for add_fb_python_executable(); forces TYPE dir (C extensions cannot load from zip) + post-build symlinks thrift/python/ and folly/ into bundled output directory Reviewed By: createdbysk Differential Revision: D98036616 fbshipit-source-id: 7fa4be1cb4a3495b66672e2a69f0b47aa6b32e92
1 parent bcd6b93 commit 2341342

2 files changed

Lines changed: 327 additions & 0 deletions

File tree

‎build/fbcode_builder/CMake/FBThriftPythonLibrary.cmake‎

Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
3+
include(FBCMakeParseArgs)
4+
include(FBPythonBinary)
5+
6+
# ---------------------------------------------------------------------------
7+
# thrift-python runtime discovery
8+
# ---------------------------------------------------------------------------
9+
# thrift_types.py (generated by mstch_python) needs two C-extension packages
10+
# at runtime:
11+
# 1. thrift.python – from the pip-installed fbthrift-python wheel
12+
# 2. folly – from folly-python site-packages
13+
#
14+
# Because these are C extensions (.so), they CANNOT be loaded from inside a
15+
# Python zip archive. Any executable that transitively depends on a
16+
# thrift-python library MUST use TYPE dir (not the default zipapp).
17+
#
18+
# Call fb_find_thrift_python_runtime() once at configure time to locate the
19+
# site-packages directories. Then use add_fb_thrift_python_executable()
20+
# instead of add_fb_python_executable() for targets that import thrift_types.
21+
# ---------------------------------------------------------------------------
22+
23+
macro(fb_find_thrift_python_runtime)
24+
if(NOT DEFINED _FB_THRIFT_PYTHON_RUNTIME_FOUND)
25+
# Ensure Python3 is found so we can use ${Python3_EXECUTABLE}
26+
if(NOT Python3_EXECUTABLE)
27+
find_package(Python3 COMPONENTS Interpreter REQUIRED)
28+
endif()
29+
30+
# Derive the getdeps install root (where all dependencies are installed).
31+
# getdeps sets GETDEPS_INSTALL_DIR; fall back to CMAKE_INSTALL_PREFIX/..
32+
if(DEFINED ENV{GETDEPS_INSTALL_DIR})
33+
set(_getdeps_root "$ENV{GETDEPS_INSTALL_DIR}")
34+
else()
35+
get_filename_component(_getdeps_root "${CMAKE_INSTALL_PREFIX}/.." ABSOLUTE)
36+
endif()
37+
message(STATUS "getdeps install root: ${_getdeps_root}")
38+
39+
# --- thrift.python (from pip-installed fbthrift-python wheel) ---
40+
# Detect via import thrift.python.types — this module only exists in
41+
# fbthrift-python, not in Apache Thrift, so it's an unambiguous check.
42+
# If not available, auto-install the wheel from getdeps build output.
43+
execute_process(
44+
COMMAND "${Python3_EXECUTABLE}" -c
45+
"import thrift.python.types; import thrift, os; print(os.path.dirname(os.path.dirname(thrift.__file__)))"
46+
OUTPUT_VARIABLE THRIFT_PYTHON_SITE_PACKAGES
47+
OUTPUT_STRIP_TRAILING_WHITESPACE
48+
ERROR_QUIET
49+
RESULT_VARIABLE _thrift_result
50+
)
51+
if(NOT _thrift_result EQUAL 0 OR NOT THRIFT_PYTHON_SITE_PACKAGES)
52+
# Not pip-installed yet — auto-install from getdeps build output
53+
file(GLOB _fbthrift_wheels
54+
"${_getdeps_root}/fbthrift-python/share/thrift/wheels/*.whl"
55+
)
56+
if(_fbthrift_wheels)
57+
list(GET _fbthrift_wheels 0 _fbthrift_whl)
58+
message(STATUS "Auto-installing fbthrift-python wheel: ${_fbthrift_whl}")
59+
execute_process(
60+
COMMAND "${Python3_EXECUTABLE}" -m pip install --no-deps "${_fbthrift_whl}"
61+
RESULT_VARIABLE _pip_result
62+
)
63+
if(_pip_result EQUAL 0)
64+
execute_process(
65+
COMMAND "${Python3_EXECUTABLE}" -c
66+
"import thrift.python.types; import thrift, os; print(os.path.dirname(os.path.dirname(thrift.__file__)))"
67+
OUTPUT_VARIABLE THRIFT_PYTHON_SITE_PACKAGES
68+
OUTPUT_STRIP_TRAILING_WHITESPACE
69+
ERROR_QUIET
70+
RESULT_VARIABLE _thrift_result
71+
)
72+
endif()
73+
endif()
74+
endif()
75+
76+
if(THRIFT_PYTHON_SITE_PACKAGES)
77+
message(STATUS "Found thrift-python runtime: ${THRIFT_PYTHON_SITE_PACKAGES}")
78+
endif()
79+
if(NOT THRIFT_PYTHON_SITE_PACKAGES)
80+
message(WARNING
81+
"thrift-python runtime not found. "
82+
"Searched: ${_getdeps_root}/fbthrift-python/share/thrift/wheels/*.whl "
83+
"pip install the fbthrift-python wheel before configuring the project.")
84+
endif()
85+
86+
# --- folly (from folly-python site-packages) ---
87+
# folly-python doesn't produce a wheel; it installs directly to
88+
# a site-packages directory under its getdeps prefix.
89+
execute_process(
90+
COMMAND "${Python3_EXECUTABLE}" -c
91+
"import folly, os; print(os.path.dirname(os.path.dirname(folly.__file__)))"
92+
OUTPUT_VARIABLE FOLLY_PYTHON_SITE_PACKAGES
93+
OUTPUT_STRIP_TRAILING_WHITESPACE
94+
ERROR_QUIET
95+
RESULT_VARIABLE _folly_result
96+
)
97+
if(NOT _folly_result EQUAL 0)
98+
# Search the getdeps install prefix for folly-python site-packages
99+
file(GLOB _folly_sp_dirs
100+
"${_getdeps_root}/folly-python/lib64/python*/site-packages"
101+
"${_getdeps_root}/folly-python/lib/python*/site-packages"
102+
)
103+
foreach(_sp IN LISTS _folly_sp_dirs)
104+
if(EXISTS "${_sp}/folly/__init__.py")
105+
set(FOLLY_PYTHON_SITE_PACKAGES "${_sp}")
106+
break()
107+
endif()
108+
endforeach()
109+
if(FOLLY_PYTHON_SITE_PACKAGES)
110+
message(STATUS "Found folly-python runtime (via prefix): ${FOLLY_PYTHON_SITE_PACKAGES}")
111+
else()
112+
message(WARNING
113+
"folly-python runtime not found. "
114+
"Searched: ${_getdeps_root}/folly-python/lib*/python*/site-packages "
115+
"Ensure folly-python is built.")
116+
endif()
117+
else()
118+
message(STATUS "Found folly-python runtime: ${FOLLY_PYTHON_SITE_PACKAGES}")
119+
endif()
120+
121+
set(_FB_THRIFT_PYTHON_RUNTIME_FOUND TRUE CACHE INTERNAL "thrift-python runtime discovery completed")
122+
set(THRIFT_PYTHON_SITE_PACKAGES "${THRIFT_PYTHON_SITE_PACKAGES}" CACHE INTERNAL "thrift-python site-packages path")
123+
set(FOLLY_PYTHON_SITE_PACKAGES "${FOLLY_PYTHON_SITE_PACKAGES}" CACHE INTERNAL "folly-python site-packages path")
124+
endif()
125+
endmacro()
126+
127+
# ---------------------------------------------------------------------------
128+
# add_fbthrift_python_library()
129+
# ---------------------------------------------------------------------------
130+
# Generate a thrift-python library from a thrift file.
131+
# Analogous to add_fbthrift_py_library() but uses the mstch_python generator
132+
# (--gen mstch_python) which produces thrift_types.py instead of ttypes.py.
133+
#
134+
# Unlike add_fbthrift_py_library(), this does NOT depend on FBThrift::thrift_py
135+
# (the legacy pure-Python runtime). The thrift-python runtime (thrift.python
136+
# and folly) are C extensions resolved at runtime via symlinks — see
137+
# add_fb_thrift_python_executable().
138+
# ---------------------------------------------------------------------------
139+
function(add_fbthrift_python_library LIB_NAME THRIFT_FILE)
140+
# Parse the arguments
141+
set(one_value_args NAMESPACE THRIFT_INCLUDE_DIR)
142+
set(multi_value_args SERVICES DEPENDS OPTIONS)
143+
fb_cmake_parse_args(
144+
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
145+
)
146+
147+
if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR)
148+
set(ARG_THRIFT_INCLUDE_DIR "include/thrift-files")
149+
endif()
150+
151+
get_filename_component(base ${THRIFT_FILE} NAME_WE)
152+
set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}-python")
153+
154+
# Parse the namespace value
155+
if (NOT DEFINED ARG_NAMESPACE)
156+
set(ARG_NAMESPACE "${base}")
157+
endif()
158+
159+
# mstch_python outputs to gen-python/<namespace>/
160+
string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
161+
set(py_output_dir "${output_dir}/gen-python/${namespace_dir}")
162+
list(APPEND generated_sources
163+
"${py_output_dir}/thrift_types.py"
164+
"${py_output_dir}/thrift_enums.py"
165+
"${py_output_dir}/thrift_metadata.py"
166+
"${py_output_dir}/thrift_abstract_types.py"
167+
"${py_output_dir}/thrift_mutable_types.py"
168+
)
169+
foreach(service IN LISTS ARG_SERVICES)
170+
list(APPEND generated_sources
171+
"${py_output_dir}/thrift_clients.py"
172+
"${py_output_dir}/thrift_mutable_clients.py"
173+
"${py_output_dir}/thrift_services.py"
174+
"${py_output_dir}/thrift_mutable_services.py"
175+
)
176+
break() # Service files are per-module, not per-service
177+
endforeach()
178+
179+
# Define a dummy interface library to help propagate the thrift include
180+
# directories between dependencies.
181+
add_library("${LIB_NAME}.thrift_includes" INTERFACE)
182+
target_include_directories(
183+
"${LIB_NAME}.thrift_includes"
184+
INTERFACE
185+
"$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>"
186+
"$<INSTALL_INTERFACE:${ARG_THRIFT_INCLUDE_DIR}>"
187+
)
188+
foreach(dep IN LISTS ARG_DEPENDS)
189+
target_link_libraries(
190+
"${LIB_NAME}.thrift_includes"
191+
INTERFACE "${dep}.thrift_includes"
192+
)
193+
endforeach()
194+
195+
# This generator expression gets the list of include directories required
196+
# for all of our dependencies.
197+
if (NOT POLICY CMP0067)
198+
message(FATAL_ERROR "add_fbthrift_python_library() requires CMake 3.8+")
199+
endif()
200+
set(
201+
thrift_include_options
202+
"-I;$<JOIN:$<TARGET_PROPERTY:${LIB_NAME}.thrift_includes,INTERFACE_INCLUDE_DIRECTORIES>,;-I;>"
203+
)
204+
205+
string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}")
206+
207+
# mstch_python generates output at gen-python/<namespace_py3>/<basename>/
208+
# The CMake NAMESPACE parameter must match this full path (including the
209+
# basename appended by the generator). For example, phy.thrift with
210+
# "namespace py3 neteng.fboss.phy" produces gen-python/neteng/fboss/phy/phy/
211+
# so NAMESPACE must be "neteng.fboss.phy.phy".
212+
213+
# Emit the rule to run the thrift compiler
214+
add_custom_command(
215+
OUTPUT
216+
${generated_sources}
217+
COMMAND_EXPAND_LISTS
218+
COMMAND
219+
"${CMAKE_COMMAND}" -E make_directory "${output_dir}"
220+
COMMAND
221+
"${FBTHRIFT_COMPILER}"
222+
--gen "mstch_python:${GEN_ARG_STR}"
223+
"${thrift_include_options}"
224+
-I "${FBTHRIFT_INCLUDE_DIR}"
225+
-o "${output_dir}"
226+
"${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}"
227+
WORKING_DIRECTORY
228+
"${CMAKE_BINARY_DIR}"
229+
MAIN_DEPENDENCY
230+
"${THRIFT_FILE}"
231+
DEPENDS
232+
"${FBTHRIFT_COMPILER}"
233+
)
234+
235+
# Register as a python library.
236+
# No FBThrift::thrift_py here — thrift-python does not use the legacy
237+
# pure-Python thrift runtime. The C-extension runtime (thrift.python,
238+
# folly) is resolved at runtime by add_fb_thrift_python_executable().
239+
add_fb_python_library(
240+
"${LIB_NAME}"
241+
BASE_DIR "${output_dir}/gen-python"
242+
NAMESPACE ""
243+
SOURCES ${generated_sources}
244+
DEPENDS ${ARG_DEPENDS}
245+
)
246+
endfunction()
247+
248+
# ---------------------------------------------------------------------------
249+
# add_fb_thrift_python_executable()
250+
# ---------------------------------------------------------------------------
251+
# Drop-in replacement for add_fb_python_executable() for targets that
252+
# transitively import thrift_types.py.
253+
#
254+
# Differences from add_fb_python_executable():
255+
# 1. Forces TYPE dir (C extensions cannot load from zip archives)
256+
# 2. Post-build: symlinks thrift/python/ from the pip-installed
257+
# fbthrift-python wheel into the bundled thrift/ directory so that
258+
# both thrift.Thrift (legacy) and thrift.python (new) coexist
259+
# 3. Post-build: symlinks folly/ from folly-python site-packages
260+
#
261+
# Usage is identical to add_fb_python_executable():
262+
# add_fb_thrift_python_executable(
263+
# my_target
264+
# MAIN_MODULE my.module:main
265+
# SOURCES ${SRCS}
266+
# DEPENDS some_py_lib FBThrift::thrift_py
267+
# )
268+
# ---------------------------------------------------------------------------
269+
function(add_fb_thrift_python_executable TARGET)
270+
fb_find_thrift_python_runtime()
271+
272+
# Forward all arguments, injecting TYPE dir
273+
add_fb_python_executable(
274+
${TARGET}
275+
TYPE dir
276+
${ARGN}
277+
)
278+
279+
set(target_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}")
280+
281+
# Post-build symlinks resolve C extension imports at runtime.
282+
# Expected directory layout after build:
283+
# <target>/
284+
# ├── __main__.py
285+
# ├── thrift/
286+
# │ ├── __init__.py
287+
# │ └── python/ → <pip-site-packages>/thrift/python/
288+
# ├── folly/ → <getdeps-site-packages>/folly/
289+
# ├── <your_module>/... (Python sources)
290+
# └── <namespace>/<module>/thrift_types.py (generated thrift-python)
291+
#
292+
# Since we no longer depend on FBThrift::thrift_py (the legacy pure-Python
293+
# runtime), there may be no bundled thrift/ directory yet. Create it
294+
# with an __init__.py so that "import thrift.python" works.
295+
if(THRIFT_PYTHON_SITE_PACKAGES AND EXISTS "${THRIFT_PYTHON_SITE_PACKAGES}/thrift/python")
296+
add_custom_command(
297+
TARGET ${TARGET}.GEN_PY_EXE POST_BUILD
298+
COMMAND ${CMAKE_COMMAND} -E make_directory "${target_dir}/thrift"
299+
COMMAND ${CMAKE_COMMAND} -E touch "${target_dir}/thrift/__init__.py"
300+
COMMAND ${CMAKE_COMMAND} -E create_symlink
301+
"${THRIFT_PYTHON_SITE_PACKAGES}/thrift/python"
302+
"${target_dir}/thrift/python"
303+
COMMENT "Symlinking thrift.python C extensions into ${TARGET}"
304+
)
305+
else()
306+
message(WARNING
307+
"${TARGET}: thrift.python not found at "
308+
"${THRIFT_PYTHON_SITE_PACKAGES}/thrift/python — "
309+
"thrift_types.py imports will fail at runtime")
310+
endif()
311+
312+
# Symlink folly C extensions (folly.iobuf etc.)
313+
if(FOLLY_PYTHON_SITE_PACKAGES AND EXISTS "${FOLLY_PYTHON_SITE_PACKAGES}/folly")
314+
add_custom_command(
315+
TARGET ${TARGET}.GEN_PY_EXE POST_BUILD
316+
COMMAND ${CMAKE_COMMAND} -E create_symlink
317+
"${FOLLY_PYTHON_SITE_PACKAGES}/folly"
318+
"${target_dir}/folly"
319+
COMMENT "Symlinking folly-python C extensions into ${TARGET}"
320+
)
321+
else()
322+
message(WARNING
323+
"${TARGET}: folly-python not found — "
324+
"thrift_types.py imports will fail at runtime")
325+
endif()
326+
endfunction()

‎build/fbcode_builder/manifests/libevent-python‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ EVENT__DISABLE_BENCHMARK = ON
3232
EVENT__DISABLE_SAMPLES = ON
3333
EVENT__DISABLE_REGRESS = ON
3434
CMAKE_POSITION_INDEPENDENT_CODE=ON
35+
EVENT__LIBRARY_TYPE = STATIC
3536

3637
[cmake.defines.shared_libs=on]
3738
EVENT__BUILD_SHARED_LIBRARIES = ON

0 commit comments

Comments
 (0)