Skip to content

Commit 9b0c034

Browse files
junpufjunpuf
authored
vllm + rayserve (#5403)
1 parent c2716a5 commit 9b0c034

File tree

3 files changed

+126
-1
lines changed

3 files changed

+126
-1
lines changed

‎vllm/buildspec-rayserve.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
2+
prod_account_id: &PROD_ACCOUNT_ID 763104351884
3+
region: &REGION <set-$REGION-in-environment>
4+
framework: &FRAMEWORK vllm
5+
version: &VERSION "0.10.2"
6+
short_version: &SHORT_VERSION "0.10"
7+
arch_type: &ARCH_TYPE x86_64
8+
autopatch_build: "False"
9+
10+
repository_info:
11+
build_repository: &BUILD_REPOSITORY
12+
image_type: &IMAGE_TYPE gpu
13+
root: .
14+
repository_name: &REPOSITORY_NAME !join [ pr, "-", *FRAMEWORK ]
15+
repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/, *REPOSITORY_NAME ]
16+
release_repository_name: &RELEASE_REPOSITORY_NAME !join [ *FRAMEWORK ]
17+
release_repository: &RELEASE_REPOSITORY !join [ *PROD_ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/, *RELEASE_REPOSITORY_NAME ]
18+
19+
context:
20+
build_context: &BUILD_CONTEXT
21+
deep_learning_container:
22+
source: src/deep_learning_container.py
23+
target: deep_learning_container.py
24+
install_efa:
25+
source: scripts/install_efa.sh
26+
target: install_efa.sh
27+
dockerd_entrypoint:
28+
source: vllm/build_artifacts/dockerd_entrypoint.sh
29+
target: dockerd_entrypoint.sh
30+
31+
images:
32+
vllm_rayserve_x86_64:
33+
<<: *BUILD_REPOSITORY
34+
context:
35+
<<: *BUILD_CONTEXT
36+
image_size_baseline: 26000
37+
device_type: &DEVICE_TYPE gpu
38+
cuda_version: &CUDA_VERSION cu128
39+
python_version: &DOCKER_PYTHON_VERSION py3
40+
tag_python_version: &TAG_PYTHON_VERSION py312
41+
os_version: &OS_VERSION ubuntu22.04
42+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-rayserve-ec2" ]
43+
latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-rayserve-ec2" ]
44+
docker_file: !join [ *FRAMEWORK, /, *ARCH_TYPE, /, *DEVICE_TYPE, /Dockerfile.rayserve ]
45+
target: vllm-rayserve-ec2
46+
build: true
47+
enable_common_stage_build: false
48+
test_configs:
49+
test_platforms:
50+
- sanity
51+
- security
52+
- ec2
53+
- eks

‎vllm/x86_64/gpu/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,11 @@ RUN apt-get upgrade && \
6565
apt-get upgrade -y && \
6666
apt-get clean
6767

68-
# Install EFA
6968
COPY dockerd_entrypoint.sh /usr/local/bin/dockerd_entrypoint.sh
7069
RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh
7170

7271
ENTRYPOINT ["/usr/local/bin/dockerd_entrypoint.sh"]
72+
7373
# ====================== sagemaker =========================================
7474
FROM base AS vllm-sagemaker
7575

‎vllm/x86_64/gpu/Dockerfile.rayserve‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
FROM docker.io/vllm/vllm-openai:v0.10.2 as base
2+
ARG PYTHON="python3"
3+
LABEL maintainer="Amazon AI"
4+
ARG EFA_VERSION="1.43.3"
5+
LABEL dlc_major_version="1"
6+
ENV DEBIAN_FRONTEND=noninteractive \
7+
LANG=C.UTF-8 \
8+
LC_ALL=C.UTF-8 \
9+
DLC_CONTAINER_TYPE=base \
10+
# Python won’t try to write .pyc or .pyo files on the import of source modules
11+
# Force stdin, stdout and stderr to be totally unbuffered. Good for logging
12+
PYTHONDONTWRITEBYTECODE=1 \
13+
PYTHONUNBUFFERED=1 \
14+
PYTHONIOENCODING=UTF-8 \
15+
LD_LIBRARY_PATH="/usr/local/lib:/opt/amazon/ofi-nccl/lib/x86_64-linux-gnu:/opt/amazon/openmpi/lib:/opt/amazon/efa/lib:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" \
16+
PATH="/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/cuda/bin:${PATH}"
17+
18+
WORKDIR /
19+
20+
COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py
21+
COPY bash_telemetry.sh /usr/local/bin/bash_telemetry.sh
22+
COPY install_efa.sh install_efa.sh
23+
24+
RUN chmod +x /usr/local/bin/deep_learning_container.py && \
25+
chmod +x /usr/local/bin/bash_telemetry.sh && \
26+
echo 'source /usr/local/bin/bash_telemetry.sh' >> /etc/bash.bashrc && \
27+
# OSS compliance - use Python zipfile instead of unzip
28+
HOME_DIR=/root && \
29+
curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip && \
30+
python3 -c "import zipfile, os; zipfile.ZipFile('/root/oss_compliance.zip').extractall('/root/'); os.remove('/root/oss_compliance.zip')" && \
31+
cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance && \
32+
chmod +x /usr/local/bin/testOSSCompliance && \
33+
chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh && \
34+
${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} && \
35+
# create symlink for python
36+
ln -s /usr/bin/python3 /usr/bin/python && \
37+
# clean up
38+
rm -rf ${HOME_DIR}/oss_compliance* && \
39+
rm -rf /tmp/tmp* && \
40+
rm -rf /tmp/uv* && \
41+
rm -rf /var/lib/apt/lists/* && \
42+
rm -rf /root/.cache | true
43+
44+
45+
RUN bash install_efa.sh ${EFA_VERSION} && \
46+
rm install_efa.sh && \
47+
mkdir -p /tmp/nvjpeg \
48+
&& cd /tmp/nvjpeg \
49+
&& wget https://developer.download.nvidia.com/compute/cuda/redist/libnvjpeg/linux-x86_64/libnvjpeg-linux-x86_64-12.4.0.76-archive.tar.xz \
50+
&& tar -xvf libnvjpeg-linux-x86_64-12.4.0.76-archive.tar.xz \
51+
&& rm -rf /usr/local/cuda/targets/x86_64-linux/lib/libnvjpeg* \
52+
&& rm -rf /usr/local/cuda/targets/x86_64-linux/include/nvjpeg.h \
53+
&& cp libnvjpeg-linux-x86_64-12.4.0.76-archive/lib/libnvjpeg* /usr/local/cuda/targets/x86_64-linux/lib/ \
54+
&& cp libnvjpeg-linux-x86_64-12.4.0.76-archive/include/* /usr/local/cuda/targets/x86_64-linux/include/ \
55+
&& rm -rf /tmp/nvjpeg \
56+
# remove cuobjdump and nvdisasm
57+
&& rm -rf /usr/local/cuda/bin/cuobjdump* \
58+
&& rm -rf /usr/local/cuda/bin/nvdisasm*
59+
60+
# ====================== ray serve =========================================
61+
FROM base AS vllm-rayserve-ec2
62+
63+
RUN uv pip install --system ray[serve]==2.49.0 && \
64+
uv cache clean \
65+
apt-get upgrade && \
66+
apt-get upgrade -y && \
67+
apt-get clean
68+
69+
COPY dockerd_entrypoint.sh /usr/local/bin/dockerd_entrypoint.sh
70+
RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh
71+
72+
ENTRYPOINT ["/usr/local/bin/dockerd_entrypoint.sh"]

0 commit comments

Comments
 (0)