Skip to content

Commit 15dbd88

Browse files
mehrdadnsimon-mo
authored andcommitted
Python 3.8 compatibility (#7754)
1 parent 1ab9815 commit 15dbd88

File tree

9 files changed

+87
-46
lines changed

9 files changed

+87
-46
lines changed

.bazelrc

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ build --per_file_copt="-\\.(asm|S)$,\\.pb\\.cc$@-w"
2121
build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
2222
# Ignore this warning since it's impractical to fix in the relevant headers
2323
build --per_file_copt="-\\.(asm|S)$,.*/ray/raylet/reconstruction_policy_test\\.cc@-Wno-inconsistent-missing-override"
24+
# Ignore this warning since CPython and Cython have issue removing deprecated tp_print on MacOS
25+
build --per_file_copt="-\\.(asm|S)$,.*/_raylet\\.cpp@-Wno-deprecated-declarations"
26+
build --per_file_copt="-\\.(asm|S)$,.*/_streaming\\.cpp@-Wno-deprecated-declarations"
2427
# Ignore minor warnings for host tools, which we generally can't control
2528
build --host_copt="-Wno-builtin-macro-redefined"
2629
build --host_copt="-Wno-inconsistent-missing-override"

.travis.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,13 @@ matrix:
148148
# This command should be kept in sync with ray/python/README-building-wheels.md,
149149
# except the `$MOUNT_BAZEL_CACHE` part.
150150

151-
- ./ci/suppress_output docker run -e TRAVIS_COMMIT=$TRAVIS_COMMIT --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -ti rayproject/arrow_linux_x86_64_base:ARROW-5631 /ray/python/build-wheel-manylinux1.sh
151+
- ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -e TRAVIS_COMMIT=$TRAVIS_COMMIT -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
152152

153153
script:
154154
- if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
155155

156156
# Explicitly sleep 60 seconds for logs to go through
157-
- ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false)
157+
- ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
158158
cache: false
159159

160160
# Build MacOS wheels.
@@ -172,7 +172,7 @@ matrix:
172172
- if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi
173173

174174
# Explicitly sleep 60 seconds for logs to go through
175-
- ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false)
175+
- ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
176176

177177
# RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml).
178178
- os: linux

bazel/ray_deps_setup.bzl

+2-2
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ def ray_deps_setup():
178178
auto_http_archive(
179179
name = "cython",
180180
build_file = True,
181-
url = "https://github.com/cython/cython/archive/49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e.tar.gz",
182-
sha256 = "0b697ac90d1e46842c7cbbf5f4a1bde5b7b41037c611167417115337e3756eaa",
181+
url = "https://github.com/cython/cython/archive/26cb654dcf4ed1b1858daf16b39fd13406b1ac64.tar.gz",
182+
sha256 = "d21e155ac9a455831f81608bb06620e4a1d75012a630faf11f4c25ad10cfc9bb",
183183
)
184184

185185
auto_http_archive(

build.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ set -x
66
set -e
77

88
# As the supported Python versions change, edit this array:
9-
SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" )
9+
SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" "3.8" )
1010

1111
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
1212

ci/travis/test-wheels.sh

+64-32
Original file line numberDiff line numberDiff line change
@@ -21,79 +21,111 @@ else
2121
exit 1
2222
fi
2323

24-
TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_microbenchmarks.py"
24+
TEST_DIR="$TRAVIS_BUILD_DIR/python/ray/tests"
25+
TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
2526
UI_TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_webui.py"
2627

27-
if [[ "$platform" == "linux" ]]; then
28-
# Now test Python 3.6.
28+
function retry {
29+
local n=1
30+
local max=3
31+
32+
while true; do
33+
"$@" && break || {
34+
if [[ $n -lt $max ]]; then
35+
((n++))
36+
echo "Command failed. Attempt $n/$max:"
37+
else
38+
echo "The command has failed after $n attempts."
39+
exit 1
40+
fi
41+
}
42+
done
43+
}
2944

45+
if [[ "$platform" == "linux" ]]; then
3046
# Install miniconda.
31-
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda3.sh
47+
PY_WHEEL_VERSIONS=("36" "37" "38")
48+
PY_MMS=("3.6.9"
49+
"3.7.6"
50+
"3.8.2")
51+
wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
3252
bash miniconda3.sh -b -p "$HOME/miniconda3"
53+
export PATH="$HOME/miniconda3/bin:$PATH"
54+
55+
for ((i=0; i<${#PY_MMS[@]}; ++i)); do
56+
PY_MM="${PY_MMS[i]}"
57+
PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}"
58+
59+
conda install -y python="${PY_MM}"
60+
61+
PYTHON_EXE="$HOME/miniconda3/bin/python"
62+
PIP_CMD="$HOME/miniconda3/bin/pip"
3363

34-
PYTHON_EXE=$HOME/miniconda3/bin/python
35-
PIP_CMD=$HOME/miniconda3/bin/pip
64+
# Find the right wheel by grepping for the Python version.
65+
PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
3666

37-
# Find the right wheel by grepping for the Python version.
38-
PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 '36')
67+
# Install the wheel.
68+
"$PIP_CMD" install -q "$PYTHON_WHEEL"
3969

40-
# Install the wheel.
41-
$PIP_CMD install -q "$PYTHON_WHEEL"
70+
# Check that ray.__commit__ was set properly.
71+
"$PYTHON_EXE" -u -c "import ray; print(ray.__commit__)" | grep "$TRAVIS_COMMIT" || (echo "ray.__commit__ not set properly!" && exit 1)
4272

43-
# Check that ray.__commit__ was set properly.
44-
$PYTHON_EXE -u -c "import ray; print(ray.__commit__)" | grep $TRAVIS_COMMIT || (echo "ray.__commit__ not set properly!" && exit 1)
73+
# Install the dependencies to run the tests.
74+
"$PIP_CMD" install -q aiohttp google grpcio pytest requests
4575

46-
# Install the dependencies to run the tests.
47-
$PIP_CMD install -q aiohttp google grpcio pytest requests
76+
# Run a simple test script to make sure that the wheel works.
77+
INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
4878

49-
# Run a simple test script to make sure that the wheel works.
50-
INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
51-
$PYTHON_EXE "$TEST_SCRIPT"
79+
for SCRIPT in "${TEST_SCRIPTS[@]}"; do
80+
retry "$PYTHON_EXE" "$SCRIPT"
81+
done
5282

53-
# Run the UI test to make sure that the packaged UI works.
54-
$PYTHON_EXE "$UI_TEST_SCRIPT"
83+
# Run the UI test to make sure that the packaged UI works.
84+
retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
85+
done
5586

5687
# Check that the other wheels are present.
5788
NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l)
58-
if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then
89+
if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then
5990
echo "Wrong number of wheels found."
6091
ls -l "$ROOT_DIR/../.whl/"
6192
exit 2
6293
fi
6394

6495
elif [[ "$platform" == "macosx" ]]; then
6596
MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
97+
PY_WHEEL_VERSIONS=("35" "36" "37" "38")
6698
PY_MMS=("3.5"
6799
"3.6"
68-
"3.7")
69-
# This array is just used to find the right wheel.
70-
PY_WHEEL_VERSIONS=("35"
71-
"36"
72-
"37")
100+
"3.7"
101+
"3.8")
73102

74103
for ((i=0; i<${#PY_MMS[@]}; ++i)); do
75-
PY_MM=${PY_MMS[i]}
76-
PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]}
104+
PY_MM="${PY_MMS[i]}"
105+
106+
PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}"
77107

78-
PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
108+
PYTHON_EXE="$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM"
79109
PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
80110

81111
# Find the appropriate wheel by grepping for the Python version.
82112
PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
83113

84114
# Install the wheel.
85-
$PIP_CMD install -q "$PYTHON_WHEEL"
115+
"$PIP_CMD" install -q "$PYTHON_WHEEL"
86116

87117
# Install the dependencies to run the tests.
88-
$PIP_CMD install -q aiohttp google grpcio pytest requests
118+
"$PIP_CMD" install -q aiohttp google grpcio pytest requests
89119

90120
# Run a simple test script to make sure that the wheel works.
91121
INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
92-
$PYTHON_EXE "$TEST_SCRIPT"
122+
for SCRIPT in "${TEST_SCRIPTS[@]}"; do
123+
retry "$PYTHON_EXE" "$SCRIPT"
124+
done
93125

94126
if (( $(echo "$PY_MM >= 3.0" | bc) )); then
95127
# Run the UI test to make sure that the packaged UI works.
96-
$PYTHON_EXE "$UI_TEST_SCRIPT"
128+
retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
97129
fi
98130

99131
done

python/build-wheel-macos.sh

+8-4
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,21 @@ DOWNLOAD_DIR=python_downloads
1515

1616
PY_VERSIONS=("3.5.3"
1717
"3.6.1"
18-
"3.7.0")
18+
"3.7.0"
19+
"3.8.2")
1920
PY_INSTS=("python-3.5.3-macosx10.6.pkg"
2021
"python-3.6.1-macosx10.6.pkg"
21-
"python-3.7.0-macosx10.6.pkg")
22+
"python-3.7.0-macosx10.6.pkg"
23+
"python-3.8.2-macosx10.9.pkg")
2224
PY_MMS=("3.5"
2325
"3.6"
24-
"3.7")
26+
"3.7"
27+
"3.8")
2528

2629
# The minimum supported numpy version is 1.14, see
2730
# https://issues.apache.org/jira/browse/ARROW-3141
2831
NUMPY_VERSIONS=("1.14.5"
32+
"1.14.5"
2933
"1.14.5"
3034
"1.14.5")
3135

@@ -76,7 +80,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
7680
$PIP_CMD install -q setuptools_scm==3.1.0
7781
# Fix the numpy version because this will be the oldest numpy version we can
7882
# support.
79-
$PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0
83+
$PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.15
8084
# Install wheel to avoid the error "invalid command 'bdist_wheel'".
8185
$PIP_CMD install -q wheel
8286
# Set the commit SHA in __init__.py.

python/build-wheel-manylinux1.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@ chmod +x /usr/bin/nproc
1313

1414
PYTHONS=("cp35-cp35m"
1515
"cp36-cp36m"
16-
"cp37-cp37m")
16+
"cp37-cp37m"
17+
"cp38-cp38")
1718

1819
# The minimum supported numpy version is 1.14, see
1920
# https://issues.apache.org/jira/browse/ARROW-3141
2021
NUMPY_VERSIONS=("1.14.5"
22+
"1.14.5"
2123
"1.14.5"
2224
"1.14.5")
2325

@@ -55,7 +57,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
5557
pushd python
5658
# Fix the numpy version because this will be the oldest numpy version we can
5759
# support.
58-
/opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.0
60+
/opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.15
5961
# Set the commit SHA in __init__.py.
6062
if [ -n "$TRAVIS_COMMIT" ]; then
6163
sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak

python/ray/cloudpickle/cloudpickle_fast.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ def _numpy_frombuffer(buffer, dtype, shape, order):
417417
array = _frombuffer(buffer, dtype, shape, order)
418418
# Unfortunately, numpy does not follow the standard, so we still
419419
# have to set the readonly flag for it here.
420-
array.setflags(write=not buffer.readonly)
420+
array.setflags(write=isinstance(buffer, bytearray) or not buffer.readonly)
421421
return array
422422

423423

python/setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def find_version(*filepath):
191191
# The BinaryDistribution argument triggers build_ext.
192192
distclass=BinaryDistribution,
193193
install_requires=requires,
194-
setup_requires=["cython >= 0.29"],
194+
setup_requires=["cython >= 0.29.14"],
195195
extras_require=extras,
196196
entry_points={
197197
"console_scripts": [

0 commit comments

Comments
 (0)