Skip to content

Commit 335d909

Browse files
ajagadi1ajagadi1
authored
[DL Streamer Pipeline Server] Reduce runtime docker image size and create separate images for extended runtime, tests and sources (#330)
1 parent 0433196 commit 335d909

19 files changed

Lines changed: 299 additions & 213 deletions

File tree

‎microservices/dlstreamer-pipeline-server/Dockerfile‎

Lines changed: 142 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
ARG BASE_IMAGE
2-
FROM ${BASE_IMAGE} AS dlstreamer-pipeline-server
2+
FROM ${BASE_IMAGE} AS gstudfloader-builder
33
ENV DLSTREAMER_DIR=/opt/intel/dlstreamer
44
ENV GSTREAMER_DIR=$DLSTREAMER_DIR/gstreamer
55

@@ -57,8 +57,8 @@ RUN apt-get install -y --no-install-recommends python3-dev
5757

5858
# Build UDF loader lib
5959
RUN /bin/bash -c "echo $PATH && \
60-
pip3 install numpy==1.26.4 && \
61-
pip3 install Cython==0.29.34 && \
60+
pip3 install --no-cache-dir numpy==1.26.4 && \
61+
pip3 install --no-cache-dir Cython==0.29.34 && \
6262
cd /home/pipeline-server/UDFLoader && \
6363
rm -rf build && \
6464
mkdir build && \
@@ -100,8 +100,77 @@ RUN /bin/bash -c "cd /home/pipeline-server/gst-udf-loader/ \
100100

101101
############################# DL Streamer Pipeline Server runtime ################################
102102

103+
FROM ${BASE_IMAGE} AS dlstreamer-pipeline-server
104+
105+
ENV DLSTREAMER_DIR=/opt/intel/dlstreamer
106+
ENV GSTREAMER_DIR=$DLSTREAMER_DIR/gstreamer
107+
103108
USER root
104109

110+
ENV VIRTUAL_ENV=/python3venv
111+
ENV PATH=$VIRTUAL_ENV/bin:$PATH
112+
ENV PYTHONPATH=/opt/intel/dlstreamer/gstreamer/lib/python3/dist-packages:/opt/intel/dlstreamer/python:/opt/intel/dlstreamer/gstreamer/lib/python3/dist-packages
113+
114+
WORKDIR /home/pipeline-server
115+
116+
RUN . /etc/os-release && \
117+
if [ "$VERSION_ID" = "24.04" ]; then \
118+
export no_proxy= && \
119+
apt-get update --allow-releaseinfo-change && apt-get install -y --no-install-recommends git \
120+
libglib2.0-0 \
121+
libusb-1.0-0 \
122+
libtool \
123+
zlib1g \
124+
zip \
125+
unzip \
126+
libopencv-core406 \
127+
libopencv-imgcodecs406 \
128+
libcjson1; \
129+
else \
130+
export no_proxy= && \
131+
apt-get update --allow-releaseinfo-change && apt-get install -y --no-install-recommends git \
132+
libglib2.0-0 \
133+
libusb-1.0-0 \
134+
libtool \
135+
zlib1g \
136+
zip \
137+
unzip \
138+
libopencv-core4.5d \
139+
libopencv-imgcodecs4.5d \
140+
libcjson1; \
141+
fi && \
142+
apt clean && \
143+
rm -rf /var/lib/apt/lists/*
144+
145+
ARG CMAKE_INSTALL_PREFIX
146+
ARG UTILS_LIB_VERSION
147+
ARG MSGBUS_LIB_VERSION
148+
149+
COPY packages/eii-utils-${UTILS_LIB_VERSION}-Linux.deb ${WORKDIR}
150+
COPY packages/eii-messagebus-${MSGBUS_LIB_VERSION}-Linux.deb ${WORKDIR}
151+
COPY packages/util-${UTILS_LIB_VERSION}.zip ${WORKDIR}
152+
153+
# Installation of utils, eiimessagebus and debian packages. Needed for UDFLoader build
154+
RUN dpkg -i /home/pipeline-server/eii-utils-${UTILS_LIB_VERSION}-Linux.deb && \
155+
dpkg -i /home/pipeline-server/eii-messagebus-${MSGBUS_LIB_VERSION}-Linux.deb && \
156+
rm -rf eii-*.deb
157+
158+
COPY packages/UDFLoader.zip ${WORKDIR}
159+
COPY packages/udfs.zip ${WORKDIR}
160+
161+
RUN unzip UDFLoader.zip -d /home/pipeline-server && \
162+
unzip udfs.zip -d /home/pipeline-server && \
163+
unzip util-${UTILS_LIB_VERSION}.zip -d /home/pipeline-server && \
164+
rm -rf udfs.zip UDFLoader.zip util-*.zip
165+
166+
ENV LD_LIBRARY_PATH=${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_PREFIX}/lib/udfs:${DLSTREAMER_DIR}/lib:${DLSTREAMER_DIR}/lib/gstreamer-1.0:${LD_LIBRARY_PATH}:/root/.local/bin \
167+
LIBRARY_PATH=/opt/intel/dlstreamer/gstreamer/lib/:${LIBRARY_PATH} \
168+
CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:${DLSTREAMER_DIR}/include/dlstreamer/gst/videoanalytics:${DLSTREAMER_DIR}/include/dlstreamer/gst/metadata:/root/.local/bin:${DLSTREAMER_DIR}/gstreamer/include/gstreamer-1.0/ \
169+
C_INCLUDE_PATH=${C_INCLUDE_PATH}:${DLSTREAMER_DIR}/gstreamer/include/gstreamer-1.0/ \
170+
PYTHONPATH=$PYTHONPATH:/usr/local/lib/:/root/.local/bin \
171+
PATH=$PATH:${DLSTREAMER_DIR}/gstreamer/bin:${DLSTREAMER_DIR}/gstreamer/bin/gstreamer-1.0:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/bin \
172+
PKG_CONFIG_PATH=/usr/share/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:/usr/lib/pkgconfig:/opt/intel/dlstreamer/gstreamer/lib/pkgconfig:/opt/intel/dlstreamer/build/intel64/Release/lib/pkgconfig:
173+
105174
WORKDIR /home/pipeline-server
106175

107176
ENV DEBIAN_FRONTEND=noninteractive
@@ -114,8 +183,10 @@ ENV GST_DEBUG="1"
114183

115184
ARG CMAKE_INSTALL_PREFIX
116185

117-
RUN cp ${CMAKE_INSTALL_PREFIX}/lib/libeiiudfloader.so ${DLSTREAMER_DIR}/gstreamer/lib
118-
RUN cp /home/pipeline-server/gst-udf-loader/build/gst_plugin/libgstudfloader.so ${DLSTREAMER_DIR}/gstreamer/lib
186+
COPY --from=gstudfloader-builder ${CMAKE_INSTALL_PREFIX}/lib/udf.so ${DLSTREAMER_DIR}/gstreamer/lib
187+
COPY --from=gstudfloader-builder ${CMAKE_INSTALL_PREFIX}/lib/libeiiudfloader.so ${DLSTREAMER_DIR}/gstreamer/lib
188+
COPY --from=gstudfloader-builder /home/pipeline-server/gst-udf-loader/build/gst_plugin/libgstudfloader.so ${DLSTREAMER_DIR}/gstreamer/lib
189+
COPY --from=gstudfloader-builder ${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}
119190

120191
ARG USER
121192
ARG UID
@@ -135,132 +206,11 @@ ENV XDG_RUNTIME_DIR=/home/.xdg_runtime_dir
135206
RUN mkdir -p $XDG_RUNTIME_DIR && chmod -R g+s $XDG_RUNTIME_DIR && chown ${USER}:users $XDG_RUNTIME_DIR
136207

137208
ENV LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/local/lib:/home/pipeline-server:${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_PREFIX}/lib/udfs
138-
139-
### To install other/newer Genicam camera SDKs add the installation steps here
140-
141-
RUN apt-get update && apt-get install -y --no-install-recommends git
142-
143209
ENV PATH=$PATH:/home/pipeline-server/.local/bin
144210

145-
COPY ./tests/requirements.txt /home/pipeline-server/tests/requirements.txt
146-
RUN pip3 install -r /home/pipeline-server/tests/requirements.txt
147-
148-
# Install for ntp timestamping
149-
RUN pip3 install ntplib==0.4.0
150-
151-
#Patch and install OpenVINO model api
152-
COPY ./docker/model_api.patch /home/pipeline-server/model_api.patch
153-
RUN pip3 install openvino-model-api==0.2.5
154-
155-
# site packages path independent of Python version (Python 3.10 for Ubuntu 22.04 and Python 3.12 for Ubuntu 24.04)
156-
RUN MODEL_API_PATH=$(python3 -c "import site; print(site.getsitepackages()[0])")/model_api && \
157-
cd $MODEL_API_PATH && \
158-
git apply /home/pipeline-server/model_api.patch
159-
160-
# Ubuntu 22: Python 3.10 still often gets pre-built datumaro wheels, so Rust might be unused.
161-
# Ubuntu 24: Python 3.12 rarely has wheels for datumaro, so it falls back to source — Rust is required here (for building datumaro - needed for geti-sdk installation).
162-
# Detect Ubuntu version and install Rust only on Ubuntu 24
163-
RUN . /etc/os-release && \
164-
if [ "$VERSION_ID" = "24.04" ]; then \
165-
apt-get update && \
166-
apt-get install -y --no-install-recommends curl build-essential && \
167-
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.87.0 && \
168-
echo 'source $HOME/.cargo/env' >> ~/.bashrc && \
169-
export PATH="/root/.cargo/bin:$PATH" && \
170-
pip install --upgrade pip && \
171-
pip install geti-sdk==2.7.1 && \
172-
# clean up Rust installation files
173-
rm -rf /root/.cargo /root/.rustup; \
174-
else \
175-
pip install --upgrade pip && \
176-
pip install geti-sdk==2.7.1; \
177-
fi && \
178-
apt-get clean && \
179-
rm -rf /var/lib/apt/lists/* /tmp/* /root/.cache
180-
181-
182-
183-
# Uninstall cuda-python installed by Geti SDK because of proprietary license causing OSPDT issue
184-
RUN pip3 uninstall -y cuda-python aiohappyeyeballs
185-
186-
# Install schedule for python job scheduling
187-
RUN pip3 install schedule==1.2.1
188-
189-
# Install opcua
190-
RUN pip3 install asyncua==1.1.5
191-
192-
# Downgrading Flask due to an atttribute error(JSONEncoder)
193-
RUN pip3 uninstall -y Flask && \
194-
pip3 install Flask==2.2.5
195-
196-
# Visualizer webtrc requirements
197-
RUN apt-get update && apt-get install libnice10 libnice-dev -y --no-install-recommends
198-
#RUN cp /usr/lib/x86_64-linux-gnu/gstreamer-1.0/libgstnice.so /opt/intel/dlstreamer/lib/gstreamer-1.0
199-
200-
WORKDIR /thirdparty
201-
202-
USER root
203-
RUN pip3 install pydantic==2.8.2
204-
205-
RUN apt-get update && apt-get install libxtst6 -y --no-install-recommends
206-
RUN pip3 install deep_sort_realtime
207-
208-
ARG DOWNLOAD_GPL_SOURCES
209-
210-
ARG UBUNTU_COPYLEFT_DEPS=""
211-
212-
ARG PYTHON_COPYLEFT_DEPS="https://git.launchpad.net/launchpadlib \
213-
https://github.com/GNOME/pygobject \
214-
https://github.com/FreeOpcUa/opcua-asyncio \
215-
https://github.com/Lucretiel/autocommand \
216-
https://github.com/certifi/python-certifi \
217-
https://git.launchpad.net/wadllib \
218-
https://git.launchpad.net/ubuntu/+source/python-apt \
219-
https://git.launchpad.net/lazr.restfulclient \
220-
https://git.launchpad.net/lazr.uri"
221-
222-
ARG PYTHON_NO_REPO_SOURCE="https://files.pythonhosted.org/packages/32/12/0409b3992c9a023d1521d9352d4c41bb1d43684ccb82899e716103e2bd88/bubblewrap-1.2.0.zip"
223-
224-
COPY ./thirdparty/third_party_deb_apk_deps.txt /thirdparty/
225-
COPY ./thirdparty/third_party_programs.txt /thirdparty/
226-
227-
RUN if [ "$DOWNLOAD_GPL_SOURCES" = "yes" ]; then \
228-
sed -Ei 's/# deb-src /deb-src /' /etc/apt/sources.list && \
229-
apt-get update && \
230-
root_dir=$PWD && \
231-
mkdir -p ./apt-sources/dlstreamer-pipeline-server && cd ./apt-sources/dlstreamer-pipeline-server && \
232-
cp ../../third_party_deb_apk_deps.txt . && \
233-
for line in $(cat third_party_deb_apk_deps.txt | xargs -n1); \
234-
do \
235-
package=$(echo $line); \
236-
grep -l GPL /usr/share/doc/${package}/copyright; \
237-
exit_status=$?; \
238-
if [ $exit_status -eq 0 ]; then \
239-
apt-get source -q --download-only $package; \
240-
fi \
241-
done && \
242-
cd $root_dir && \
243-
echo "Cloning Debian and Ubuntu github deps..." && \
244-
mkdir -p ./github-sources/Ubuntu_Deb && cd ./github-sources/Ubuntu_Deb && \
245-
for f in `echo $UBUNTU_COPYLEFT_DEPS | xargs -n1`; do git clone $f && \
246-
cd "$(basename "$f")" && \
247-
rm -rf .git && \
248-
cd ..; done && \
249-
cd ../ && \
250-
mkdir -p Python && cd Python && \
251-
echo "Cloning Python github dependency" && \
252-
for f in `echo $PYTHON_COPYLEFT_DEPS | xargs -n1`; do git clone $f && \
253-
wget $PYTHON_NO_REPO_SOURCE && \
254-
cd "$(basename "$f")" && \
255-
rm -rf .git && \
256-
cd ..; done && \
257-
cd $root_dir && \
258-
echo "Download source for $(ls | wc -l) third-party packages: $(du -sh)"; \
259-
rm -rf /var/lib/apt/lists/*;\
260-
fi
261-
262-
263-
WORKDIR /home/pipeline-server
211+
RUN apt-get update && apt-get install libxtst6 -y --no-install-recommends && \
212+
apt clean && \
213+
rm -rf /var/lib/apt/lists/*
264214

265215
USER $USER
266216

@@ -269,19 +219,22 @@ COPY ./docker/run.sh .
269219
COPY ./utils/*.py ./utils/
270220
COPY ./src/ ./src
271221

272-
# Install server requirements
273222
USER root
274-
RUN pip3 install PyYAML==5.4.1 --no-build-isolation && \
275-
pip3 install -r /home/pipeline-server/src/server/requirements.service.txt \
276-
-r /home/pipeline-server/src/server/requirements.webrtc.txt \
277-
-r /home/pipeline-server/src/server/requirements.txt && \
278-
python3 -m pip install --upgrade Werkzeug==3.1.3 orjson==3.10.12 packaging==24.2 boto3==1.36.17 influxdb-client==1.48.0 marshmallow==4.0.0
279-
USER ${USER}
280223

281-
# OpenTelemetry
282-
USER root
283-
RUN pip3 install --no-cache -r /home/pipeline-server/src/opentelemetry/requirements.txt
284-
RUN pip3 install --upgrade opentelemetry-exporter-otlp
224+
# Cleanup
225+
RUN apt-get remove -y zip unzip git && \
226+
apt-get clean && \
227+
rm -rf /var/lib/apt/lists/*
228+
229+
RUN pip3 install --no-cache-dir --upgrade pip && \
230+
pip3 uninstall --no-cache-dir -y Flask && \
231+
# Downgrading Flask due to an atttribute error(JSONEncoder)
232+
pip3 install --no-cache-dir Flask==2.2.5 && \
233+
pip3 install --no-cache-dir PyYAML==5.4.1 --no-build-isolation && \
234+
pip3 install --no-cache-dir -r /home/pipeline-server/src/requirements.txt && \
235+
pip3 install --no-cache-dir --upgrade Werkzeug==3.1.3 && \
236+
pip3 install --upgrade opentelemetry-exporter-otlp
237+
285238
USER ${USER}
286239

287240
# Disable OpenVINO Telemetry
@@ -296,16 +249,6 @@ COPY ./user_scripts/udfs/python/add_label.py /home/pipeline-server/udfs/python/a
296249
COPY ./user_scripts/udfs/python/dummy_publisher.py /home/pipeline-server/udfs/python/dummy_publisher.py
297250
COPY ./user_scripts/udfs/python/geti_udf/ /home/pipeline-server/udfs/python/geti_udf/
298251

299-
# Copy unit tests
300-
COPY ./tests /home/pipeline-server/tests
301-
302-
# Copy geti pallet defect detection sample
303-
COPY ./resources/models/geti /home/pipeline-server/resources/models/geti
304-
COPY ./resources/videos/classroom.avi /home/pipeline-server/resources/videos/classroom.avi
305-
COPY ./resources/videos/warehouse.avi /home/pipeline-server/resources/videos/warehouse.avi
306-
COPY ./resources/videos/person-bicycle-car-detection.mp4 /home/pipeline-server/resources/videos/person-bicycle-car-detection.mp4
307-
#COPY ./pipelines/user_defined_pipelines/person_detection /home/pipeline-server/pipelines/user_defined_pipelines/person_detection
308-
309252
# Copy GVAPYTHON samples
310253
COPY ./user_scripts/gvapython/geti/ /home/pipeline-server/gvapython/geti
311254
COPY ./user_scripts/gvapython/mqtt_publisher/ /home/pipeline-server/gvapython/mqtt_publisher
@@ -317,4 +260,52 @@ COPY ./configs/default/config.json /home/pipeline-server/config.json
317260

318261
ENV PYTHONPATH=$PYTHONPATH:/home/pipeline-server/udfs/python/geti_udf
319262

320-
ENTRYPOINT ["./run.sh"]
263+
ENTRYPOINT ["./run.sh"]
264+
265+
266+
############################# DL Streamer Pipeline Server extended runtime ################################
267+
268+
FROM dlstreamer-pipeline-server AS dlstreamer-pipeline-server-extended
269+
270+
USER root
271+
272+
# Patch and install OpenVINO model api
273+
COPY ./docker/model_api.patch /home/pipeline-server/model_api.patch
274+
RUN pip3 install --no-cache-dir openvino-model-api==0.2.5
275+
276+
# site packages path independent of Python version (Python 3.10 for Ubuntu 22.04 and Python 3.12 for Ubuntu 24.04)
277+
RUN apt-get update && \
278+
apt-get install -y --no-install-recommends git && \
279+
MODEL_API_PATH=$(python3 -c "import site; print(site.getsitepackages()[0])")/model_api && \
280+
cd $MODEL_API_PATH && \
281+
git apply /home/pipeline-server/model_api.patch && \
282+
apt-get remove -y git && \
283+
apt-get clean && \
284+
rm -rf /var/lib/apt/lists/*
285+
286+
# Ubuntu 22: Python 3.10 still often gets pre-built datumaro wheels, so Rust might be unused.
287+
# Ubuntu 24: Python 3.12 rarely has wheels for datumaro, so it falls back to source — Rust is required here (for building datumaro - needed for geti-sdk installation).
288+
# Detect Ubuntu version and install Rust only on Ubuntu 24
289+
RUN . /etc/os-release && \
290+
if [ "$VERSION_ID" = "24.04" ]; then \
291+
apt-get update && \
292+
apt-get install -y --no-install-recommends curl build-essential && \
293+
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.87.0 && \
294+
echo 'source $HOME/.cargo/env' >> ~/.bashrc && \
295+
export PATH="/root/.cargo/bin:$PATH" && \
296+
pip3 install --no-cache-dir --upgrade pip && \
297+
pip3 install --no-cache-dir geti-sdk==2.7.1 && \
298+
# clean up Rust installation files
299+
rm -rf /root/.cargo /root/.rustup; \
300+
else \
301+
pip3 install --no-cache-dir --upgrade pip && \
302+
pip3 install --no-cache-dir geti-sdk==2.7.1; \
303+
fi && \
304+
apt-get clean && \
305+
rm -rf /var/lib/apt/lists/* /tmp/* /root/.cache
306+
307+
308+
# Uninstall cuda-python installed by Geti SDK because of proprietary license causing OSPDT issue
309+
RUN pip3 uninstall --no-cache-dir -y cuda-python aiohappyeyeballs
310+
311+
USER ${USER}

0 commit comments

Comments
 (0)