Skip to content

chore: Update Triton + Ray Serve Tutorial for Ray Summit 2024 #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Sep 26, 2024
Merged

chore: Update Triton + Ray Serve Tutorial for Ray Summit 2024 #115

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
6feb95e
updating with minor changes
nnshah1 Sep 17, 2024
9cd5089
updating to latest triton and trtsdk and moving to stable diffusion xl
nnshah1 Sep 18, 2024
9069f47
updating
nnshah1 Sep 18, 2024
4c04569
updating with torch compile baseline
nnshah1 Sep 19, 2024
859ab80
updating scaling parameters - now baseline and triton scale
nnshah1 Sep 19, 2024
2cd53e8
updating deployment names - shortening
nnshah1 Sep 25, 2024
e275c64
Updating readme for latest versions
nnshah1 Sep 26, 2024
a8ea8ad
updated information about huggingface login for stablediffusion 1.5
nnshah1 Sep 26, 2024
2758f26
updating readme for current versions
nnshah1 Sep 26, 2024
0ac3bd3
updating readme
nnshah1 Sep 26, 2024
9eaf98d
updated
nnshah1 Sep 26, 2024
2256d9a
removing unneeded files and flags
nnshah1 Sep 26, 2024
d415407
fix pre-commit
nnshah1 Sep 26, 2024
56e3d91
updating copyright
nnshah1 Sep 26, 2024
58260c4
updating typo fix
nnshah1 Sep 26, 2024
655a71a
updating and removing PWD from volume mount
nnshah1 Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Popular_Models_Guide/StableDiffusion/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile

# Base Images
BASE_IMAGE=nvcr.io/nvidia/tritonserver
BASE_IMAGE_TAG_DIFFUSION=24.01-py3
BASE_IMAGE_TAG_DIFFUSION=24.08-py3

get_options() {
while :; do
Expand Down Expand Up @@ -141,7 +141,7 @@ get_options() {
fi

if [ -z "$TAG" ]; then
TAG="tritonserver:r24.01"
TAG="tritonserver:r24.08"

if [[ $FRAMEWORK == "DIFFUSION" ]]; then
TAG+="-diffusion"
Expand Down Expand Up @@ -211,7 +211,7 @@ if [[ $FRAMEWORK == DIFFUSION ]]; then
set -x
fi
$RUN_PREFIX mkdir -p $PWD/backend/diffusion
$RUN_PREFIX docker run --rm -it -v $PWD:/workspace $TAG /bin/bash -c "cp -rf /tmp/TensorRT/demo/Diffusion /workspace/backend/diffusion"
$RUN_PREFIX docker run --rm -it -v ${SOURCE_DIR}:/workspace $TAG /bin/bash -c "cp -rf /tmp/TensorRT/demo/Diffusion /workspace/backend/diffusion"

{ set +x; } 2>/dev/null

Expand All @@ -221,7 +221,7 @@ if [[ $FRAMEWORK == DIFFUSION ]]; then
set -x
fi

$RUN_PREFIX docker run --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/build_models.sh --model $model"
$RUN_PREFIX docker run --rm -it -v ${SOURCE_DIR):/workspace $TAG /bin/bash -c "/workspace/scripts/build_models.sh --model $model"

{ set +x; } 2>/dev/null
done
Expand Down
4 changes: 2 additions & 2 deletions Popular_Models_Guide/StableDiffusion/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ ARG BASE_IMAGE_TAG=24.01-py3

FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as tritonserver-stable-diffusion

RUN pip install --pre --upgrade --extra-index-url https://pypi.nvidia.com tensorrt==9.2.0.post12.dev5
RUN pip install --pre --upgrade --extra-index-url https://pypi.nvidia.com tensorrt-cu12==10.4.0

RUN git clone https://github.com/NVIDIA/TensorRT.git -b release/9.2 --single-branch /tmp/TensorRT
RUN git clone https://github.com/NVIDIA/TensorRT.git -b release/10.4 --single-branch /tmp/TensorRT

RUN pip3 install -r /tmp/TensorRT/demo/Diffusion/requirements.txt

Expand Down
2 changes: 1 addition & 1 deletion Popular_Models_Guide/StableDiffusion/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ get_options() {
fi

if [ -z "$IMAGE" ]; then
IMAGE="tritonserver:r24.01"
IMAGE="tritonserver:r24.08"

if [[ $FRAMEWORK == "DIFFUSION" ]]; then
IMAGE+="-diffusion"
Expand Down
33 changes: 17 additions & 16 deletions Triton_Inference_Server_Python_API/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile

# Base Images
BASE_IMAGE=nvcr.io/nvidia/tritonserver
BASE_IMAGE_TAG_IDENTITY=24.01-py3
BASE_IMAGE_TAG_DIFFUSION=24.01-py3
BASE_IMAGE_TAG_IDENTITY=24.08-py3
BASE_IMAGE_TAG_DIFFUSION=24.08-py3
BASE_IMAGE_TAG_TRT_LLM=24.01-trtllm-python-py3

get_options() {
Expand Down Expand Up @@ -138,7 +138,7 @@ get_options() {
fi

if [ -z "$TAG" ]; then
TAG="triton-python-api:r24.01"
TAG="triton-python-api:r24.08"

if [[ $FRAMEWORK == "TRT_LLM" ]]; then
TAG+="-trt-llm"
Expand Down Expand Up @@ -186,7 +186,7 @@ get_options "$@"

if [[ $FRAMEWORK == DIFFUSION ]]; then
BASE_IMAGE="tritonserver"
BASE_IMAGE_TAG="r24.01-diffusion"
BASE_IMAGE_TAG="r24.08-diffusion"
fi

# BUILD RUN TIME IMAGE
Expand All @@ -207,17 +207,18 @@ if [[ $FRAMEWORK == DIFFUSION ]]; then
if [ -z "$RUN_PREFIX" ]; then
set -x
fi
$RUN_PREFIX mkdir -p backend/diffusion
$RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag tritonserver:r24.01-diffusion
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/backend/diffusion/model.py backend/diffusion/model.py
$RUN_PREFIX mkdir -p diffusion-models/stable_diffusion_1_5/1
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/config.pbtxt diffusion-models/stable_diffusion_1_5/config.pbtxt
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/1/.gitkeep diffusion-models/stable_diffusion_1_5/1/.gitkeep
$RUN_PREFIX mkdir -p diffusion-models/stable_diffusion_xl/1
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_xl/config.pbtxt diffusion-models/stable_diffusion_xl/config.pbtxt
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_xl/1/.gitkeep diffusion-models/stable_diffusion_xl/1/.gitkeep
$RUN_PREFIX mkdir -p scripts/stable_diffusion
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/scripts/build_models* scripts/stable_diffusion/
$RUN_PREFIX mkdir -p ${SOURCE_DIR}/backend/diffusion
$RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag tritonserver:r24.08-diffusion
$RUN_PREFIX docker run --rm -it -v ${SOURCE_DIR}:/workspace tritonserver:r24.08-diffusion /bin/bash -c "cp -rf /tmp/TensorRT/demo/Diffusion /workspace/backend/diffusion"
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/backend/diffusion/model.py ${SOURCE_DIR}/backend/diffusion/model.py
$RUN_PREFIX mkdir -p ${SOURCE_DIR}/diffusion-models/stable_diffusion_1_5/1
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/config.pbtxt ${SOURCE_DIR}/diffusion-models/stable_diffusion_1_5/config.pbtxt
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/1/.gitkeep ${SOURCE_DIR}/diffusion-models/stable_diffusion_1_5/1/.gitkeep
$RUN_PREFIX mkdir -p ${SOURCE_DIR}/diffusion-models/stable_diffusion_xl/1
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_xl/config.pbtxt ${SOURCE_DIR}/diffusion-models/stable_diffusion_xl/config.pbtxt
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_xl/1/.gitkeep ${SOURCE_DIR}/diffusion-models/stable_diffusion_xl/1/.gitkeep
$RUN_PREFIX mkdir -p ${SOURCE_DIR}/scripts/stable_diffusion
$RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/scripts/build_models* ${SOURCE_DIR}/scripts/stable_diffusion/

fi

Expand Down Expand Up @@ -249,7 +250,7 @@ if [[ $FRAMEWORK == IDENTITY ]] || [[ $BUILD_MODELS == TRUE ]]; then
set -x
fi

$RUN_PREFIX docker run --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_1_5"
$RUN_PREFIX docker run --gpus all --rm -it -v ${SOURCE_DIR}:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_xl"

{ set +x; } 2>/dev/null
fi
Expand Down
10 changes: 1 addition & 9 deletions Triton_Inference_Server_Python_API/deps/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,6 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

awscli
fastapi==0.97.0
ftfy
mypy
pyright
pytest
ray[all]==2.9
scipy
sphinx
sphinx-markdown-builder
starlette==0.27.0
ray[all]==2.36.0
Binary file removed Triton_Inference_Server_Python_API/deps/tritonserver-2.41.0.dev0-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
28 changes: 9 additions & 19 deletions Triton_Inference_Server_Python_API/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,37 +25,27 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver
ARG BASE_IMAGE_TAG=24.01-py3
ARG BASE_IMAGE_TAG=24.08-py3

FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api

RUN apt-get update; apt-get install -y gdb

COPY ./deps/requirements.txt /tmp/requirements.txt

RUN pip install --timeout=2000 -r /tmp/requirements.txt
RUN --mount=type=bind,source=./deps/requirements.txt,target=/tmp/requirements.txt \
pip install --timeout=2000 --requirement /tmp/requirements.txt

# Finish pyright install

RUN pyright --help

COPY ./deps/tritonserver-2.41.0.dev0-py3-none-any.whl /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl

RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \
"tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all]
"tritonserver-*.whl" | xargs -I {} pip3 install --upgrade {}[all]

RUN pip3 show tritonserver 1>/dev/null || \
if [ $? != 0 ]; then \
pip3 install /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl[all] ;\
fi
# grafana
RUN apt-get install -y adduser libfontconfig1 musl && \
wget https://dl.grafana.com/enterprise/release/grafana-enterprise_11.2.0_amd64.deb && \
dpkg -i grafana-enterprise_11.2.0_amd64.deb && \
rm -rf grafana-enterprise_11.2.0_amd64.deb

RUN ln -sf /bin/bash /bin/sh

COPY . /workspace

ARG RUN_TESTS=FALSE

RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r23.12-python-api https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi

RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi

74 changes: 0 additions & 74 deletions Triton_Inference_Server_Python_API/docker/Dockerfile.trt-llm-engine-builder
View file
Open in desktop

This file was deleted.

14 changes: 14 additions & 0 deletions Triton_Inference_Server_Python_API/examples/rayserve/start_ray.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

ip_address=$(hostname -I | awk '{print $1}')

echo $ip_address

mkdir -p /tmp/rayserve-demo; cd /tmp/rayserve-demo

ray metrics launch-prometheus

export RAY_GRAFANA_HOST=http://${ip_address}:3000

ray start --head --dashboard-host 0.0.0.0 --metrics-export-port 8080 --disable-usage-stats

/usr/share/grafana/bin/grafana-server --homepath /usr/share/grafana --config /tmp/ray/session_latest/metrics/grafana/grafana.ini web >grafana.stdout.log 2>&1 &
3 changes: 3 additions & 0 deletions Triton_Inference_Server_Python_API/examples/rayserve/stop_ray.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
ray stop
pkill prometheus.*
pkill grafana.*
Loading
Loading