Skip to content

Commit 8178d4f

Browse files
tuhaihetuhaihe
committed
Replace pip3 download with curl for Python deps
Replace pip3 download commands with curl for downloading psutil, PyYAML, PyGreSQL and mock packages in gpMgmt/bin/Makefile. This change unifies the download mechanism to use curl consistently for all Python dependencies, avoiding potential pip3 download compatibility issues across different environments. Changes: - Remove pip3 download commands for psutil, PyYAML, PyGreSQL & mock - Use curl to download packages directly from PyPI - Maintain existing wheel and cython installation for PyYAML build - Keep the same package versions and download locations Also, add curl and pip3 dependency detection in configure.ac file for better user experience. When `--with-pythonsrc-ext` option is enabled, configure can check the required dependencies early.
1 parent 5cd9379 commit 8178d4f

File tree

3 files changed

+179
-21
lines changed

3 files changed

+179
-21
lines changed

‎configure‎

Lines changed: 144 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,9 @@ with_krb_srvnam
742742
krb_srvtab
743743
with_gssapi
744744
with_pythonsrc_ext
745+
PIP3
746+
CURL
747+
PYTHON3
745748
with_python
746749
with_perl
747750
with_tcl
@@ -9846,6 +9849,142 @@ fi
98469849
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
98479850
$as_echo "$with_pythonsrc_ext" >&6; }
98489851

9852+
# Check for required dependencies when pythonsrc-ext is enabled
9853+
if test "$with_pythonsrc_ext" = yes; then
9854+
# Extract the first word of "python3", so it can be a program name with args.
9855+
set dummy python3; ac_word=$2
9856+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9857+
$as_echo_n "checking for $ac_word... " >&6; }
9858+
if ${ac_cv_path_PYTHON3+:} false; then :
9859+
$as_echo_n "(cached) " >&6
9860+
else
9861+
case $PYTHON3 in
9862+
[\\/]* | ?:[\\/]*)
9863+
ac_cv_path_PYTHON3="$PYTHON3" # Let the user override the test with a path.
9864+
;;
9865+
*)
9866+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9867+
for as_dir in $PATH
9868+
do
9869+
IFS=$as_save_IFS
9870+
test -z "$as_dir" && as_dir=.
9871+
for ac_exec_ext in '' $ac_executable_extensions; do
9872+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9873+
ac_cv_path_PYTHON3="$as_dir/$ac_word$ac_exec_ext"
9874+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9875+
break 2
9876+
fi
9877+
done
9878+
done
9879+
IFS=$as_save_IFS
9880+
9881+
;;
9882+
esac
9883+
fi
9884+
PYTHON3=$ac_cv_path_PYTHON3
9885+
if test -n "$PYTHON3"; then
9886+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON3" >&5
9887+
$as_echo "$PYTHON3" >&6; }
9888+
else
9889+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9890+
$as_echo "no" >&6; }
9891+
fi
9892+
9893+
9894+
if test -z "$PYTHON3"; then
9895+
as_fn_error $? "python3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9896+
fi
9897+
9898+
# Extract the first word of "curl", so it can be a program name with args.
9899+
set dummy curl; ac_word=$2
9900+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9901+
$as_echo_n "checking for $ac_word... " >&6; }
9902+
if ${ac_cv_path_CURL+:} false; then :
9903+
$as_echo_n "(cached) " >&6
9904+
else
9905+
case $CURL in
9906+
[\\/]* | ?:[\\/]*)
9907+
ac_cv_path_CURL="$CURL" # Let the user override the test with a path.
9908+
;;
9909+
*)
9910+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9911+
for as_dir in $PATH
9912+
do
9913+
IFS=$as_save_IFS
9914+
test -z "$as_dir" && as_dir=.
9915+
for ac_exec_ext in '' $ac_executable_extensions; do
9916+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9917+
ac_cv_path_CURL="$as_dir/$ac_word$ac_exec_ext"
9918+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9919+
break 2
9920+
fi
9921+
done
9922+
done
9923+
IFS=$as_save_IFS
9924+
9925+
;;
9926+
esac
9927+
fi
9928+
CURL=$ac_cv_path_CURL
9929+
if test -n "$CURL"; then
9930+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CURL" >&5
9931+
$as_echo "$CURL" >&6; }
9932+
else
9933+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9934+
$as_echo "no" >&6; }
9935+
fi
9936+
9937+
9938+
if test -z "$CURL"; then
9939+
as_fn_error $? "curl is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9940+
fi
9941+
9942+
# Extract the first word of "pip3", so it can be a program name with args.
9943+
set dummy pip3; ac_word=$2
9944+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9945+
$as_echo_n "checking for $ac_word... " >&6; }
9946+
if ${ac_cv_path_PIP3+:} false; then :
9947+
$as_echo_n "(cached) " >&6
9948+
else
9949+
case $PIP3 in
9950+
[\\/]* | ?:[\\/]*)
9951+
ac_cv_path_PIP3="$PIP3" # Let the user override the test with a path.
9952+
;;
9953+
*)
9954+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9955+
for as_dir in $PATH
9956+
do
9957+
IFS=$as_save_IFS
9958+
test -z "$as_dir" && as_dir=.
9959+
for ac_exec_ext in '' $ac_executable_extensions; do
9960+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9961+
ac_cv_path_PIP3="$as_dir/$ac_word$ac_exec_ext"
9962+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9963+
break 2
9964+
fi
9965+
done
9966+
done
9967+
IFS=$as_save_IFS
9968+
9969+
;;
9970+
esac
9971+
fi
9972+
PIP3=$ac_cv_path_PIP3
9973+
if test -n "$PIP3"; then
9974+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PIP3" >&5
9975+
$as_echo "$PIP3" >&6; }
9976+
else
9977+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9978+
$as_echo "no" >&6; }
9979+
fi
9980+
9981+
9982+
if test -z "$PIP3"; then
9983+
as_fn_error $? "pip3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9984+
fi
9985+
fi
9986+
9987+
98499988

98509989

98519990
#
@@ -18442,7 +18581,7 @@ else
1844218581
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1844318582
since some C++ compilers masquerading as C compilers
1844418583
incorrectly reject 9223372036854775807. */
18445-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18584+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1844618585
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1844718586
&& LARGE_OFF_T % 2147483647 == 1)
1844818587
? 1 : -1];
@@ -18488,7 +18627,7 @@ else
1848818627
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1848918628
since some C++ compilers masquerading as C compilers
1849018629
incorrectly reject 9223372036854775807. */
18491-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18630+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1849218631
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1849318632
&& LARGE_OFF_T % 2147483647 == 1)
1849418633
? 1 : -1];
@@ -18512,7 +18651,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1851218651
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1851318652
since some C++ compilers masquerading as C compilers
1851418653
incorrectly reject 9223372036854775807. */
18515-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18654+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1851618655
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1851718656
&& LARGE_OFF_T % 2147483647 == 1)
1851818657
? 1 : -1];
@@ -18557,7 +18696,7 @@ else
1855718696
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1855818697
since some C++ compilers masquerading as C compilers
1855918698
incorrectly reject 9223372036854775807. */
18560-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18699+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1856118700
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1856218701
&& LARGE_OFF_T % 2147483647 == 1)
1856318702
? 1 : -1];
@@ -18581,7 +18720,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1858118720
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1858218721
since some C++ compilers masquerading as C compilers
1858318722
incorrectly reject 9223372036854775807. */
18584-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18723+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1858518724
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1858618725
&& LARGE_OFF_T % 2147483647 == 1)
1858718726
? 1 : -1];

‎configure.ac‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,6 +1061,25 @@ AC_SUBST(with_python)
10611061
AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
10621062
PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
10631063
AC_MSG_RESULT([$with_pythonsrc_ext])
1064+
1065+
# Check for required dependencies when pythonsrc-ext is enabled
1066+
if test "$with_pythonsrc_ext" = yes; then
1067+
AC_PATH_PROG([PYTHON3], [python3])
1068+
if test -z "$PYTHON3"; then
1069+
AC_MSG_ERROR([python3 is required for --with-pythonsrc-ext but was not found])
1070+
fi
1071+
1072+
AC_PATH_PROG([CURL], [curl])
1073+
if test -z "$CURL"; then
1074+
AC_MSG_ERROR([curl is required for --with-pythonsrc-ext but was not found])
1075+
fi
1076+
1077+
AC_PATH_PROG([PIP3], [pip3])
1078+
if test -z "$PIP3"; then
1079+
AC_MSG_ERROR([pip3 is required for --with-pythonsrc-ext but was not found])
1080+
fi
1081+
fi
1082+
10641083
AC_SUBST(with_pythonsrc_ext)
10651084

10661085

‎gpMgmt/bin/Makefile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,14 @@ PYYAML_VERSION=5.4.1
9090
download-python-deps:
9191
@echo "--- Downloading Python dependencies for gpMgmt modules"
9292
@mkdir -p $(PYLIB_SRC_EXT)
93-
# Download psutil
94-
pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
95-
# Download PyYAML, wheel is for metadata vefiry;
96-
# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
97-
pip3 install wheel "cython<3.0.0"
98-
pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
99-
# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
100-
PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
93+
# Download psutil using curl
94+
curl -sSL https://files.pythonhosted.org/packages/source/p/psutil/psutil-$(PSUTIL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/psutil-$(PSUTIL_VERSION).tar.gz
95+
# Download PyYAML using curl
96+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyYAML/PyYAML-$(PYYAML_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyYAML-$(PYYAML_VERSION).tar.gz
97+
# Download PyGreSQL using curl
98+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyGreSQL/PyGreSQL-$(PYGRESQL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyGreSQL-$(PYGRESQL_VERSION).tar.gz
99+
# Install wheel and cython for PyYAML building
100+
pip3 install --user wheel "cython<3.0.0"
101101

102102
#
103103
# PyGreSQL
@@ -148,14 +148,14 @@ $(MOCK_BIN):
148148
@echo "--- mock for platform $(UBUNTU_PLATFORM)"
149149
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
150150
pip3 install mock;\
151-
else\
152-
mkdir -p $(PYLIB_SRC_EXT) && \
153-
pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
154-
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155-
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156-
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157-
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158-
fi;
151+
else\
152+
mkdir -p $(PYLIB_SRC_EXT) && \
153+
curl -sSL https://files.pythonhosted.org/packages/source/m/mock/mock-$(MOCK_VERSION).zip -o $(PYLIB_SRC_EXT)/mock-$(MOCK_VERSION).zip && \
154+
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155+
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156+
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157+
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158+
fi;
159159

160160
# -----------------------------------------------------------------------------
161161
#

0 commit comments

Comments
 (0)