From 0acf682496f852c6515a39bd8e62f706288e4d3a Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Tue, 2 Jun 2020 22:19:02 +0000 Subject: [PATCH 01/25] ignore *.ts *.m3u8 .vscode venv --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 2fae51c9..2870d48f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ __pycache__/ *.caffemodel *.uff *.avi +*.ts +*.m3u8 +.vscode +venv/ +*.tbz2 \ No newline at end of file From 1a876e804a32d03b3417a0bee36fa2c32ba9b6bc Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 4 Jun 2020 18:38:03 +0000 Subject: [PATCH 02/25] ignore video, vscode, venv --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.dockerignore b/.dockerignore index e27dc136..afa965fb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,7 @@ frontend/build/ frontend/node_modules/ experiments/ tasks/ +*.avi +*.tbz2 +.vscode/ +venv/ From b4ec7e6faaf077db2ec10f9ee873d422662c07f7 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 5 Jun 2020 20:42:22 +0000 Subject: [PATCH 03/25] WIP for merge into gstreamer --- .dockerignore | 4 +- deepstream-x86.Dockerfile | 82 +++ deepstream.ini | 41 ++ deepstream_docker_build.sh | 73 +++ libs/detectors/deepstream/__init__.py | 59 ++ libs/detectors/deepstream/_base_detector.py | 194 ++++++ libs/detectors/deepstream/_detectors.py | 122 ++++ libs/detectors/deepstream/_ds_config.py | 342 ++++++++++ libs/detectors/deepstream/_ds_engine.py | 203 ++++++ libs/detectors/deepstream/_ds_utils.py | 122 ++++ libs/detectors/deepstream/_gst_engine.py | 666 ++++++++++++++++++++ libs/detectors/deepstream/_pyds.py | 83 +++ libs/distance_pb2.py | 259 ++++++++ neuralet-distancing.py | 61 +- requirements.in | 8 + requirements.txt | 200 ++++++ test.Dockerfile | 3 + 17 files changed, 2502 insertions(+), 20 deletions(-) create mode 100644 deepstream-x86.Dockerfile create mode 100644 deepstream.ini create mode 100755 deepstream_docker_build.sh create mode 100644 libs/detectors/deepstream/__init__.py create mode 100644 libs/detectors/deepstream/_base_detector.py create mode 100644 libs/detectors/deepstream/_detectors.py create mode 100644 libs/detectors/deepstream/_ds_config.py create mode 100644 libs/detectors/deepstream/_ds_engine.py create mode 100644 libs/detectors/deepstream/_ds_utils.py create mode 100644 libs/detectors/deepstream/_gst_engine.py create mode 100644 libs/detectors/deepstream/_pyds.py create mode 100644 libs/distance_pb2.py create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 test.Dockerfile diff --git a/.dockerignore b/.dockerignore index afa965fb..259832a7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,12 +2,12 @@ demo.gif docs/ .git/ -data/ frontend/build/ frontend/node_modules/ experiments/ tasks/ *.avi -*.tbz2 .vscode/ venv/ +*.ts +*.m3u8 \ No newline at end of file diff --git a/deepstream-x86.Dockerfile b/deepstream-x86.Dockerfile new file mode 100644 index 00000000..7f5ae4db --- /dev/null +++ b/deepstream-x86.Dockerfile @@ -0,0 +1,82 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +FROM registry.hub.docker.com/mdegans/gstcudaplugin:latest + +# this can't be downloaded directly because a license needs to be accepted, +# (because those who abuse it will care so much about that) and a tarball +# extracted. This is very un-fun: +# https://developer.nvidia.com/deepstream-getting-started#python_bindings +ARG DS_PYBIND_TBZ2='ds_pybind_v0.9.tbz2' +ARG DS_SOURCES_ROOT='/opt/nvidia/deepstream/deepstream/sources' + +# copy stuff we need at the start of the build +COPY ${DS_PYBIND_TBZ2} requirements.txt /tmp/ + +# extract and install the python bindings +RUN mkdir -p ${DS_SOURCES_ROOT} \ + && tar -xf /tmp/${DS_PYBIND_TBZ2} -C ${DS_SOURCES_ROOT} + +# install pip, install requirements, remove pip and deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-gi \ + python3-gst-1.0 \ + python3-pip \ + python3-setuptools \ + python3-opencv \ + python3-dev \ + graphviz \ + && pip3 install --require-hashes -r /tmp/requirements.txt \ + && apt-get purge -y --autoremove \ + python3-pip \ + python3-setuptools \ + python3-dev \ + && rm -rf /var/lib/apt/cache/* + +# TODO(mdegans) python3-opencv brings in a *ton* of dependencies so +# it's probably better off removed from the deepstream image + +# NOTE(mdegans): these layers are here because docker's multi-line +# copy syntax is dumb and doesn't support copying folders in a sane way. +# one way of getting around this is to use a subdir for your +# project + +WORKDIR /repo + +COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /srv/frontend +COPY neuralet-distancing.py README.md ${CONFIG_FILE} ./ +COPY libs ./libs/ +COPY ui ./ui/ +COPY tools ./tools/ +COPY logs ./logs/ +COPY data ./data/ + +# drop all caps to a regular user +RUN useradd -md /var/smart_distancing -rUs /bin/false smart_distancing \ + && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/gstreamer +USER smart_distancing:smart_distancing + +# copy frontend +COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /srv/frontend + +# entrypoint with deepstream. +EXPOSE 8000 +ENTRYPOINT [ "/usr/bin/python3", "neuralet-distancing.py" ] +CMD [ "--config", "deepstream.ini" ] diff --git a/deepstream.ini b/deepstream.ini new file mode 100644 index 00000000..dd668a8c --- /dev/null +++ b/deepstream.ini @@ -0,0 +1,41 @@ +[App] +Host: 0.0.0.0 +Port: 8000 +Resolution: 640,480 +; public uri without the trailing slash +PublicUrl: http://localhost:8000 +Encoder: nvvideoconvert ! nvv4l2h264enc + +[Source_0] +; VideoPath may be a uri supported by uridecodebin (rtsp, http, etc.) +; or a local file. +; TODO(mdegans): camera sources. +VideoPath: /repo/data/TownCentreXVID.avi + +[Source_1] +VideoPath: /opt/nvidia/deepstream/deepstream-5.0/samples/streams/sample_720p.h264 + +[Detector] +; Supported devices: Deepstream +Device: DeepStream +; Detector's Name can be "resnet10" or "peoplenet" +Name: resnet10 +;ImageSize is not needed since this is included in the deepstream config .ini +ClassID: 0 +MinScore: 0.25 + +; TODO(mdegans): remove unused sections and keys from this file + +[PostProcessor] +MaxTrackFrame: 5 +NMSThreshold: 0.98 +; distance threshold for smart distancing in (cm) +DistThreshold: 150 +; ditance mesurement method, CenterPointsDistance: compare center of pedestrian boxes together, FourCornerPointsDistance: compare four corresponding points of pedestrian boxes and get the minimum of them. +DistMethod: CenterPointsDistance + +[Logger] +; options: csv, json (default is csv if not set) +Name: csv +; optional log path (default to ~/.smart_distancing/logs/): +;LogDirectory: /some/path diff --git a/deepstream_docker_build.sh b/deepstream_docker_build.sh new file mode 100755 index 00000000..60088abb --- /dev/null +++ b/deepstream_docker_build.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -e + +# change this to your docker hub user if you fork this and want to push it +readonly USER_NAME="neuralet" +# DeepStream constants: +readonly DS_PYBIND_TBZ2="ds_pybind_v0.9.tbz2" # deepstrem python bindings +readonly DS_PYBIND_URL="https://developer.nvidia.com/deepstream-getting-started#python_bindings" +# Dockerfile names +readonly X86_DOCKERFILE="deepstream-x86.Dockerfile" +readonly TEGRA_DOCKERFILE="deepstream-tegra.Dockerfile" +# https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ +readonly THIS_SCRIPT_BASENAME="${0##*/}" + +# get the docker tag suffix from the git branch +# if master, use "latest" +TAG_SUFFIX=$(git rev-parse --abbrev-ref HEAD) +if [[ $TAG_SUFFIX == "master" ]]; then + TAG_SUFFIX="deepstream" +fi + +function check_deps() { + if [ ! -f ${DS_PYBIND_TBZ2} ]; then + echo "ERROR: ${DS_PYBIND_TBZ2} needed in same directory as Dockerfile." > /dev/stderr + echo "Download from: ${DS_PYBIND_URL}" > /dev/stderr + echo "(it's inside deepstream_python_v0.9.tbz2)" > /dev/stderr + exit 1 + fi +} + +function x86() { + exec docker build -f $X86_DOCKERFILE -t "$USER_NAME/smart-distancing:$TAG_SUFFIX" . +} + +function tegra() { + exec docker build -f $TEGRA_DOCKERFILE -t "$USER_NAME/smart-distancing:$TAG_SUFFIX" . +} + +main() { + check_deps +case "$1" in + x86) + x86 + ;; + tegra) + tegra + ;; + *) + echo "Usage: $THIS_SCRIPT_BASENAME {x86|tegra}" +esac +} + +main "$1" \ No newline at end of file diff --git a/libs/detectors/deepstream/__init__.py b/libs/detectors/deepstream/__init__.py new file mode 100644 index 00000000..85d17892 --- /dev/null +++ b/libs/detectors/deepstream/__init__.py @@ -0,0 +1,59 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +The DeepStream detector module includes a DeepStream specific implementation +of the BaseDetector class and various utility classes and functions. +""" + +# GStreamer needs to be imported before pyds or else there is crash on Gst.init +import gi +gi.require_version('Gst', '1.0') +gi.require_version('GLib', '2.0') +from gi.repository import ( + Gst, + GLib, +) +from libs.detectors.deepstream._base_detector import * +from libs.detectors.deepstream._ds_utils import * +from libs.detectors.deepstream._pyds import * +from libs.detectors.deepstream._ds_config import * +from libs.detectors.deepstream._gst_engine import * +from libs.detectors.deepstream._ds_engine import * +from libs.detectors.deepstream._detectors import * + +__all__ = [ + 'BaseDetector', # _base_detector.py + 'bin_to_pdf', # _ds_utils.py + 'DsConfig', # _ds_config.py + 'DsDetector', # _detectors.py + 'DsEngine', # _ds_engine.py + 'ElemConfig', # _ds_config.py + 'find_deepstream', # _ds_utils.py + 'frame_meta_iterator', # _ds_engine.py + 'GstConfig', # _ds_config.py + 'GstEngine', # _ds_engine.py + 'link_many', # _ds_engine.py + 'obj_meta_iterator', # _ds_engine.py + 'OnFrameCallback', # _base_detector.py + 'PYDS_INSTRUCTIONS', # _pyds.py + 'PYDS_PATH', # _pyds.py + 'pyds', # _pyds.py +] diff --git a/libs/detectors/deepstream/_base_detector.py b/libs/detectors/deepstream/_base_detector.py new file mode 100644 index 00000000..eb43a152 --- /dev/null +++ b/libs/detectors/deepstream/_base_detector.py @@ -0,0 +1,194 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Contains Detector, the base class for all detectors (in the deepstream branch, anyway). +""" + +import abc +import logging +import os +import sys +import urllib.parse +import urllib.request +import itertools + +from libs.distance_pb2 import ( + BBox, + Frame, + Person, +) +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, +) + +OnFrameCallback = Callable[[Frame], Any] +Callable.__doc__ = """ +Signature an on-frame callback accepting a fra. +""" + +__all__ = ['BaseDetector', 'OnFrameCallback'] + +class BaseDetector(abc.ABC): + """ + A base class for all Detectors. The following should be overridden: + + PLATFORM the model platform (eg. edgetpu, jetson, x86) + DEFAULT_MODEL_FILE with the desired model filename + DEFAULT_MODEL_URL with the url path minus filename of the model + + load_model() to load the model. This is called for you on __init__. + + Something should also call on_frame() with a sequence of sd.Detection + + Arguments: + config (:obj:`sd.core.ConfigEngine`): + the global config class + on_frame (:obj:`OnFrameCallback`): + A callback to call on every frame. + """ + + PLATFORM = None # type: Tuple + DEFAULT_MODEL_FILE = None # type: str + DEFAULT_MODEL_URL = None # type: str + + # this works differently in the deepstream branch + MODEL_DIR = '/repo/data' + + def __init__(self, config, on_frame:OnFrameCallback=None): + # set the config + self.config = config + + # assign the on_frame callback if any + if on_frame is not None: + self.on_frame = on_frame + + # set up a logger on the class + self.logger = logging.getLogger(self.__class__.__name__) + + # download the model if necessary + if not os.path.isfile(self.model_file): + self.logger.info( + f'model does not exist under: "{self.model_path}" ' + f'downloading from "{self.model_url}"') + os.makedirs(self.model_path, mode=0o755, exist_ok=True) + urllib.request.urlretrieve(self.model_url, self.model_file) + + # add a frame counter + self._frame_count = itertools.count() + + # load the model + self.load_model() + + @property + def detector_config(self) -> Dict: + """:return: the 'Detector' section from self.config""" + return self.config.get_section_dict('Detector') + + @property + def name(self) -> str: + """:return: the detector name.""" + return self.detector_config['Name'] + + @property + def model_path(self) -> Optional[str]: + """:return: the folder containing the model.""" + try: + cfg_model_path = self.detector_config['ModelPath'] + if cfg_model_path: # not None and greater than zero in length + return cfg_model_path + except KeyError: + pass + return os.path.join(self.MODEL_DIR, self.PLATFORM) + + @property + def model_file(self) -> Optional[str]: + """:return: the model filename.""" + return os.path.join(self.model_path, self.DEFAULT_MODEL_FILE) + + @property + def model_url(self) -> str: + """:return: a parsed url pointing to a downloadable model""" + # this is done to validate it's at least a valid uri + # TODO(mdegans?): move to config class + return urllib.parse.urlunparse(urllib.parse.urlparse( + self.DEFAULT_MODEL_URL + self.DEFAULT_MODEL_FILE)) + + @property + def class_id(self) -> int: + """:return: the class id to detect.""" + return int(self.detector_config['ClassID']) + + @property + def score_threshold(self) -> float: + """:return: the detection minimum threshold (MinScore).""" + return float(self.detector_config['MinScore']) + min_score = score_threshold # an alias for ease of access + + @property + @abc.abstractmethod + def sources(self) -> List[str]: + """:return: the active sources.""" + + @sources.setter + @abc.abstractmethod + def sources(self, source: Sequence[str]): + """Set the active sources""" + + @property + @abc.abstractmethod + def fps(self) -> int: + """:return: the current fps""" + + @abc.abstractmethod + def load_model(self): + """load the model. Called by default implementation of __init__.""" + + @abc.abstractmethod + def start(self): + """ + Start the detector (should do inferences and call on_frame). + """ + pass + + @abc.abstractmethod + def stop(self): + """ + Start the detector (should do inferences and call on_frame). + """ + pass + + def on_frame(self, frame: Frame): # pylint: disable=method-hidden + """ + Calculate distances between detections and updates UI. + This default implementation just logs serialized frames to the DEBUG + level and is called if on_frame is not specified on __init__. + + Arguments: + frame (:obj:`Frame`): frame level deserialized protobuf metadata. + """ + self.logger.debug({'frame_proto': frame.SerializeToString()}) + pass diff --git a/libs/detectors/deepstream/_detectors.py b/libs/detectors/deepstream/_detectors.py new file mode 100644 index 00000000..3a053223 --- /dev/null +++ b/libs/detectors/deepstream/_detectors.py @@ -0,0 +1,122 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +DsDetector lives here +""" + +import logging +import itertools +import time + +from libs.config_engine import ConfigEngine +from libs.detectors.deepstream import ( + BaseDetector, + DsConfig, + DsEngine, + OnFrameCallback, +) + +from libs.distance_pb2 import ( + Batch, + Frame, +) + +from typing import ( + Dict, + Tuple, + Sequence, +) + +__all__ = ['DsDetector'] + +class DsDetector(BaseDetector): + """ + DeepStream implementation of BaseDetector. + """ + + DEFAULT_MODEL_FILE = 'None' + DEFAULT_MODEL_URL = 'None' + + engine = None # type: DsEngine + + def load_model(self): + """ + init/reinit a DsEngine instance (terminates if necessary). + + Called by start() automatically. + """ + if self.engine and self.engine.is_alive(): + self.logger.info( + "restarting engine") + self.engine.terminate() + self.engine.join() + self.engine = DsEngine(DsConfig(self.config)) + + # @Hossein I know the other classes don't have this, but it may make sense + # to add this start + stop functionality to the base class. + def start(self, blocking=True, timeout=10): + """ + Start DsDetector's engine. + + Arguments: + blocking (bool): + Whether to block this thread while waiting for results. If + false, busy waits with a sleep(0) in the loop. + (set False if you want this to spin) + timeout: + If blocking is True, + """ + self.logger.info( + f'starting up{" in blocking mode" if blocking else ""}') + self.engine.blocking=blocking + self.engine.start() + self.engine.queue_timeout=10 + batch = Batch() + while self.engine.is_alive(): + batch_str = self.engine.results + if not batch_str: + time.sleep(0) # this is to switch context if launched in thread + continue + batch.ParseFromString(batch_str) + for frame in batch.frames: # pylint: disable=no-member + next(self._frame_count) + self.on_frame(frame) + + def stop(self): + self.engine.stop() + + @property + def fps(self): + self.logger.warning("fps reporting not yet implemented") + return 30 + + @property + def sources(self): + self.logger.warning("getting sources at runtime not yet implemented") + return [] + + @sources.setter + def sources(self, sources: Sequence[str]): + self.logger.warning("setting sources at runtime not yet implemented") + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/libs/detectors/deepstream/_ds_config.py b/libs/detectors/deepstream/_ds_config.py new file mode 100644 index 00000000..257c15f5 --- /dev/null +++ b/libs/detectors/deepstream/_ds_config.py @@ -0,0 +1,342 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +DsConfig and GstConfig wrappers live here (they wrap ConfigEngine). +""" + +import os +import logging + +from math import ( + log, + ceil, + sqrt, +) + +import gi +gi.require_version('Gst', '1.0') +gi.require_version('GLib', '2.0') +from gi.repository import ( + Gst, +) + +from typing import ( + TYPE_CHECKING, + Any, + Tuple, + Iterable, + Mapping, + Union, + List, +) +if TYPE_CHECKING: + from libs.config_engine import ConfigEngine +else: + ConfigEngine = None + +__all__ = [ + 'DsConfig', + 'ElemConfig', + 'GstConfig', +] + +from libs.detectors.deepstream._ds_utils import find_deepstream + +Path = Union[str, os.PathLike] +ElemConfig = Mapping[str, Any] + +logger = logging.getLogger(__name__) + +def calc_rows_and_columns(num_sources: int) -> int: + """ + Calculate rows and columns values from a number of sources. + + Returns: + (int) math.ceil(math.sqrt(num_sources)) + """ + if not num_sources: + return 1 + return int(ceil(sqrt(num_sources))) + + +def calc_tile_resolution(out_res: Tuple[int, int], rows_and_columns: int) -> Tuple[int, int]: + """ + Return the optimal resolution for the stream muxer to scale input sources to. + (same as the resolution for a tile). + """ + return out_res[0] // rows_and_columns, out_res[1] // rows_and_columns + + +class GstConfig(object): + """ + GstConfig is a simple class to wrap a ConfigEngine and provide + for a GstEngine. + + Arguments: + master_config: + The master :obj:`ConfigEngine`_ to use internally. + """ + + SRC_TYPE = 'uridecodebin' + MUXER_TYPE = 'concat' # using this just because it has request pads + INFER_TYPE = 'identity' + DISTANCE_TYPE = 'identity' + PAYLOAD_TYPE = 'identity' + BROKER_TYPE = 'identity' + OSD_CONVERTER_TYPE = 'identity' + TILER_TYPE = 'identity' + OSD_TYPE = 'identity' + TRACKER_TYPE = 'identity' + + def __init__(self, master_config: ConfigEngine): + self.master_config = master_config + self.validate() + + @property + def src_configs(self) -> List[ElemConfig]: + """ + Returns: + A list containing an ElemConfig for each 'Source' Section + in self.master_config + """ + ret = [] + for section, content in self.master_config.config.items(): + if section.startswith('Source') and 'VideoPath' in content: + video_path = content['VideoPath'] + if os.path.isfile(video_path): + video_path = f'file://{os.path.abspath(video_path)}' + ret.append({ + 'uri': video_path, + }) + return ret + + @property + def class_ids(self) -> str: + """ + Returns: + the class IDs from the master config. + """ + return self.master_config.config['Detector']['ClassID'] + + @property + def infer_configs(self) -> List[ElemConfig]: + """ + Default implementation. + + Returns: + a list with a single empty :obj:`ElemConfig` + """ + return [dict(),] + + def _blank_config(self) -> ElemConfig: + """ + Default implementation. + + Returns: + a new empty :obj:`ElemConfig` + """ + return dict() + + muxer_config = property(_blank_config) + tracker_config = property(_blank_config) + tiler_config = property(_blank_config) + osd_config = property(_blank_config) + osd_converter_config = property(_blank_config) + sink_config = property(_blank_config) + distance_config = property(_blank_config) + payload_config = property(_blank_config) + broker_config = property(_blank_config) + + @property + def rows_and_columns(self) -> int: + """ + Number of rows and columns for the tiler element. + + Calculated based on the number of sources. + """ + return calc_rows_and_columns(len(self.src_configs)) + + @property + def tile_resolution(self) -> Tuple[int, int]: + """ + Resolution of an individual video tile. + + Calculated based on the resolution and number of sources. + """ + return calc_tile_resolution(self.out_resolution, self.rows_and_columns) + + @property + def out_resolution(self) -> Tuple[int, int]: + """ + Output video resolution as a 2 tuple of width, height. + + Read from self.master_config.config['App'] + """ + return tuple(int(i) for i in self.master_config.config['App']['Resolution'].split(',')) + + def validate(self): + """ + Validate `self`. Called by __init__. + + Checks: + * there is at least one source + * there is at least one inference element + + Raises: + ValueError: if `self` is invalid. + + Examples: + + If an empty source is supplied, ValueError is raised: + + >>> empty_iterable = tuple() + >>> src_configs = [{'prop': 'val'},] + >>> config = GstConfig(empty_iterable, src_configs) + Traceback (most recent call last): + ... + ValueError: at least one inference config is required + """ + if not self.infer_configs: + raise ValueError( + "at least one 'Detector' section is required in the .ini") + if not self.src_configs: + raise ValueError( + "at least one 'Source' section is required in the .ini") + + +class DsConfig(GstConfig): + """ + DeepStream implementation of GstConfig. + + 'batch-size' will may be overridden on element configs to match + the number of sources in the master config. + + Arguments: + max_batch_size (int): + The maximum allowed batch size parameter. + Defaults to 32, but this should probably be + lower on platforms like Jetson Nano for best + performance. + """ + SRC_TYPE = 'uridecodebin' + MUXER_TYPE = 'nvstreammux' + INFER_TYPE = 'nvinfer' + DISTANCE_TYPE = 'dsdistance' + PAYLOAD_TYPE = 'dsprotopayload' + BROKER_TYPE = 'payloadbroker' + OSD_CONVERTER_TYPE = 'nvvideoconvert' + TILER_TYPE = 'nvmultistreamtiler' + OSD_TYPE = 'nvdsosd' + TRACKER_TYPE = 'nvtracker' + + DS_VER, DS_ROOT = find_deepstream() + DS_CONF_PATH = os.path.join(DS_ROOT, 'samples', 'configs') + # TODO(mdegans): secure hash validation of all configs, models, paths, etc and copy to immutable path + # important that the copy is *before* the validation + RESNET_CONF = os.path.join(DS_CONF_PATH, 'deepstream-app/config_infer_primary.txt') + RESNET_CONF_NANO = os.path.join(DS_CONF_PATH, 'deepstream-app/config_infer_primary_nano.txt') + PEOPLENET_CONF = os.path.join(DS_CONF_PATH, 'tlt_pretrained_models/config_infer_primary_peoplenet.txt') + + TRACKER_LIB = 'libnvds_mot_iou.so' + INFER_INTERVAL = 1 + + def __init__(self, *args, max_batch_size=32, **kwargs): + self.max_batch_size = max_batch_size + super().__init__(*args, **kwargs) + + @property + def muxer_config(self) -> ElemConfig: + return { + 'width': self.tile_resolution[0], + 'height': self.tile_resolution[1], + 'batch-size': self.batch_size, + 'enable-padding': True, # maintain apsect raidou + 'live-source': True, + 'attach-sys-ts': True, + } + + @property + def tracker_config(self) -> ElemConfig: + return { + 'll-lib-file': os.path.join(self.DS_ROOT, 'lib', self.TRACKER_LIB), + 'enable-batch-process': True, + } + + @property + def tiler_config(self) -> ElemConfig: + return { + 'rows': self.rows_and_columns, + 'columns': self.rows_and_columns, + 'width': self.out_resolution[0], + 'height': self.out_resolution[1], + } + + @property + def infer_configs(self) -> List[ElemConfig]: + """ + Return nvinfer configs. + """ + infer_configs = [] + # TODO(mdegans): support 'Clasifier' section as secondary detectors + # this might mean parsing and writing the config files since the + # unique id is specified in the config. + detector_cfg = self.master_config.config['Detector'] + model_name = detector_cfg['Name'] + if model_name == 'resnet10': + # TODO(detect nano and use optimized cfg) + detector = { + 'config-file-path': self.RESNET_CONF, + } + elif model_name == 'peoplenet': + detector = { + 'config-file-path': self.PEOPLENET_CONF, + } + else: + raise ValueError('Invalid value for Detector "Name"') + detector['batch-size'] = self.batch_size + detector['interval'] = self.INFER_INTERVAL + infer_configs.append(detector) + return infer_configs + + @property + def batch_size(self) -> int: + """ + Return the optimal batch size. + (next power of two up from the number of sources). + + TODO(mdegans): it's unclear if this is actually optimal + and under what circumstances (depends on model, afaik) + tests must be run to see if it's better to use the number + of sources directly. + + NOTE(mdegans): Nvidia sets it to a static 30 in their config + so it may be a power of two is not optimal here. Some of + their test apps use the number of sources. Benchmarking + is probably the easiest way to settle this. + + Control the max by setting max_batch_size. + """ + optimal = pow(2, ceil(log(len(self.src_configs))/log(2))) + return min(optimal, self.max_batch_size) + +if __name__ == "__main__": + import doctest + doctest.testmod(optionflags=doctest.ELLIPSIS) diff --git a/libs/detectors/deepstream/_ds_engine.py b/libs/detectors/deepstream/_ds_engine.py new file mode 100644 index 00000000..01dbfcf0 --- /dev/null +++ b/libs/detectors/deepstream/_ds_engine.py @@ -0,0 +1,203 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +DsEngine lives here (GstEngine multprocessing.Process subclass) +""" + +import configparser +import tempfile +import logging +import queue + +# import gstreamer bidings +import gi +gi.require_version('Gst', '1.0') +gi.require_version('GLib', '2.0') +from gi.repository import ( + Gst, + GLib, +) +# import python deepstream +from libs.detectors.deepstream import pyds +# import config stuff +from libs.detectors.deepstream._ds_config import ( + GstConfig, + DsConfig, + ElemConfig, +) +# import metadata stuff +from libs.distance_pb2 import ( + Batch, + Frame, + Person, + BBox, +) +from libs.detectors.deepstream._gst_engine import GstEngine +# typing +from typing import ( + Any, + Callable, + Iterator, + Iterable, + Optional, + List, + Mapping, + TYPE_CHECKING, +) + +__all__ = [ + 'DsEngine', + 'frame_meta_iterator', + 'obj_meta_iterator', +] + +# these two functions below are used by DsEngine to parse pyds metadata + +def frame_meta_iterator(frame_meta_list: GLib.List, + reverse=False) -> Iterator[pyds.NvDsFrameMeta]: + """ + Iterate through DeepStream frame metadata GList (doubly linked list). + + Arguments: + Reverse (bool): iterate in reverse (with .previous) + """ + # generators catch StopIteration to stop iteration, + while frame_meta_list is not None: + yield pyds.glist_get_nvds_frame_meta(frame_meta_list.data) + # a Glib.List is a doubly linked list where .data is the content + # and 'next' and 'previous' contain to the next and previous elements + frame_meta_list = frame_meta_list.next if not reverse else frame_meta_list.previous + +def obj_meta_iterator(obj_meta_list: GLib.List, + reverse=False) -> Iterator[pyds.NvDsObjectMeta]: + """ + Iterate through DeepStream object metadata GList (doubly linked list). + + Arguments: + Reverse (bool): iterate in reverse (with .previous) + """ + while obj_meta_list is not None: + yield pyds.glist_get_nvds_object_meta(obj_meta_list.data) + obj_meta_list = obj_meta_list.next if not reverse else obj_meta_list.previous + +def write_config(tmpdir, config:dict) -> str: + """ + Write a nvinfer config to a .ini file in tmpdir and return the filename. + + The section heading is [property] + + Example: + >>> config = { + ... 'model-file': 'foo.caffemodel', + ... 'proto-file': 'foo.prototxt', + ... 'labelfile-path': 'foo.labels.txt', + ... 'int8-calib-file': 'foo_cal_trt.bin', + ... } + >>> with tempfile.TemporaryDirectory() as tmp: + ... filename = write_config(tmp, config) + ... print(filename) + ... with open(filename) as f: + ... for l in f: + ... print(l, end='') + /tmp/tmp.../config....ini + [property] + model-file = foo.caffemodel + proto-file = foo.prototxt + labelfile-path = foo.labels.txt + int8-calib-file = foo_cal_trt.bin + + """ + # TODO(mdegans): simple property validation to fail fast + cp = configparser.ConfigParser() + cp['property'] = config + fd, filename = tempfile.mkstemp( + prefix='config', + suffix='.ini', + dir=tmpdir, + text=True, + ) + with open(fd, 'w') as f: + cp.write(f) + return filename + +class DsEngine(GstEngine): + """ + DeepStream implemetation of GstEngine. + """ + + _tmp = None # type: tempfile.TemporaryDirectory + _previous_scores = None + + def _quit(self): + # cleanup the temporary directory we created on __init__ + self._tmp.cleanup() + # this can self terminate so it should be called last: + super()._quit() + + @property + def tmp(self): + """ + Path to the /tmp/ds_engine... folder used by this engine. + + This path is normally deleted on self._quit() + """ + return self._tmp.name + + _previous_broker_results = None + def on_buffer(self, pad: Gst.Pad, info: Gst.PadProbeInfo, _: None, ) -> Gst.PadProbeReturn: + """ + Get serialized Batch level protobuf string from self._broker + and put it in the result queue. + + connected to the tiler element's sink pad + """ + # get result, and if same as the last, skip it + # yeah, using a GObject property for this is kind + # of odd, but it works. In the future I may make a broker + # and put it in the gi.repository so it'll be easier + # to do this: + proto_str = self._broker.get_property("results") + if proto_str == self._previous_broker_results: + return Gst.PadProbeReturn.OK + self._last_on_buffer_result = proto_str + # we try to update the results queue, but it might be full if + # the results queue is full becauase the ui process is too slow + # (I haven't had this happen, but it covers this) + if not self._update_result_queue(proto_str): + # note: this can hurt performance depending on your logging + # backend (anything that blocks, which is a lot.), but really, + # if we reach this point, something is already hurting, + # and it's probably better to save the data. + self.logger.warning({'dropped_batch_proto': proto_str}) + # NOTE(mdegans): we can drop the whole buffer here if we want to drop + # entire buffers (batches, including images) along with the metadata + # return Gst.PadProbeReturn.DROP + pass + # return pad probe ok, which passes the buffer on + return Gst.PadProbeReturn.OK + + def run(self): + self._tmp = tempfile.TemporaryDirectory(prefix='ds_engine') + super().run() + +if __name__ == "__main__": + import doctest + doctest.testmod(optionflags=doctest.ELLIPSIS) diff --git a/libs/detectors/deepstream/_ds_utils.py b/libs/detectors/deepstream/_ds_utils.py new file mode 100644 index 00000000..e9caf61d --- /dev/null +++ b/libs/detectors/deepstream/_ds_utils.py @@ -0,0 +1,122 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +DeepStream common utilities. +""" + +import logging +import os +import shutil +import subprocess + +import gi +gi.require_version('Gst', '1.0') +gi.require_version('GLib', '2.0') +from gi.repository import ( + Gst, + GLib, +) + +from typing import ( + Tuple, + Optional, +) + +__all__ = [ + 'bin_to_pdf', + 'find_deepstream', +] + +DS_VERSIONS = ('4.0', '5.0') +DS4_PATH = '/opt/nvidia/deepstream/deepstream-{ver}' + +logger = logging.getLogger(__name__) + +def find_deepstream() -> Tuple[str, str]: + """ + Finds DeepStream. + + Return: + A 2 tuple of the DeepStream version + and it's root path or None if no + version is found. + """ + # TODO(mdegans): implement + for ver in DS_VERSIONS: + ds_dir = DS4_PATH.format(ver=ver) + if os.path.isdir(ds_dir): + return ver, ds_dir + +# this is from `mce.pipeline` +def bin_to_pdf(bin_: Gst.Bin, details: Gst.DebugGraphDetails, filename: str, + ) -> Optional[str]: + """ + Copied from `mce.pipeline `_ + + Dump a Gst.Bin to pdf using + `Gst.debug_bin_to_dot_file `_ + and graphviz. + Will launch the 'dot' subprocess in the background with Popen. + Does not check whether the process completes, but a .dot is + created in any case. Has the same arguments as + `Gst.debug_bin_to_dot_file `_ + + Arguments: + bin: + the bin to make a .pdf visualization of + details: + a Gst.DebugGraphDetails choice (see gstreamer docs) + filename: + a base filename to use (not full path, with no extension) + usually this is the name of the bin you can get with some_bin.name + + Returns: + the path to the created file (.dot or .pdf) or None if + GST_DEBUG_DUMP_DOT_DIR not found in os.environ + """ + if 'GST_DEBUG_DUMP_DOT_DIR' in os.environ: + dot_dir = os.environ['GST_DEBUG_DUMP_DOT_DIR'] + dot_file = os.path.join(dot_dir, f'{filename}.dot') + pdf_file = os.path.join(dot_dir, f'{filename}.pdf') + logger.debug(f"writing {bin_.name} to {dot_file}") + Gst.debug_bin_to_dot_file(bin_, details, filename) + dot_exe = shutil.which('dot') + if dot_exe: + logger.debug( + f"converting {os.path.basename(dot_file)} to " + f"{os.path.basename(pdf_file)} in background") + command = ('nohup', dot_exe, '-Tpdf', dot_file, f'-o{pdf_file}') + logger.debug( + f"running: {' '.join(command)}") + subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + preexec_fn=os.setpgrp, + ) + else: + logger.warning( + f'graphviz does not appear to be installed, so cannot convert' + f'{dot_file} to pdf. You can install graphviz with ' + f'"sudo apt install graphviz" on Linux for Tegra or Ubuntu.') + return dot_file + return pdf_file + return None diff --git a/libs/detectors/deepstream/_gst_engine.py b/libs/detectors/deepstream/_gst_engine.py new file mode 100644 index 00000000..0ea4173f --- /dev/null +++ b/libs/detectors/deepstream/_gst_engine.py @@ -0,0 +1,666 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +GstEngine lives here (multiprocessing.Process subclass). +""" + +import os +import functools +import multiprocessing +import queue +import logging + +# import gstreamer bidings +import gi +gi.require_version('Gst', '1.0') +gi.require_version('GLib', '2.0') +from gi.repository import ( + Gst, + GLib, +) +from typing import ( + Any, + Callable, + Iterable, + Optional, + Sequence, +) +from libs.detectors.deepstream._ds_utils import bin_to_pdf +from libs.detectors.deepstream._ds_config import GstConfig + +__all__ = [ + 'GstEngine', + 'GstLinkError', + 'link_many', + 'PadProbeCallback', +] + +PadProbeCallback = Callable[ + [Gst.Pad, Gst.PadProbeInfo, Any], + Gst.PadProbeReturn, +] +""" +Signature of Gsteamer Pad Probe Callback +""" + +# a documentation template for an elemetn creation function +# TODO(mdegans): remove after refactoring elem creation methods +_ELEM_DOC = """ +Create {elem_name} Gst.Element and add to the pipeline. + +Returns: + bool: False on failure, True on success. +""" + + +class GstLinkError(RuntimeError): + """on failure to link pad or element""" + +def link(a: Gst.Element, b: Gst.Element): + """ + Link Gst.Element a to b + + Use this to avoid the checking for true on exit, + which is very C, but not very Pythonic. + + (Always Availability of src and sink pads) + + Raises: + LinkError: on failure to link. + """ + if not a.link(b): + raise GstLinkError(f'could not link {a.name} to {b.name}') + +def link_many(elements: Iterable[Gst.Element]): + """ + Link many Gst.Element. + + (linear, assumes Always Availability of src and sink pads). + + Returns: + bool: False on failure, True on success. + """ + elements = iter(elements) + last = next(elements) + for current in elements: + if not last.link(current): + raise GstLinkError(f'could not link {last.name} to {current.name}') + + +class GstEngine(multiprocessing.Process): + """ + GstEngine is an internal engine for GStreamer. + + It is a subclass of multiprocessing.Process to run a GLib.MainLoop in + a separate process. There are several reasons for this: + + * GStreamer elements can have memory leaks so if and when the processes + crashes, it can be restarted without having to restart the whole app. + In general GStreamer is as buggy as it is fast and the quality of elements + runs the gamut. + * Python callbacks called by GLib.MainLoop can block the whole MainLoop. + (The same is true in C, but you can launch CPU bound stuff in a thread, + which is not possible in Python due to the GIL). Running GLib.MainLoop + it in a separate process and putting the results into a queue if a slot + is empty (dropping the results if not), avoids this problem. + * Ease of adding and removing new sources. With DeepStream, right now, the + *easiest* and most reliable way to do this is to relaunch it's process + with a modified configuration. + + Arguments: + config (:obj:`GstConfig`): + GstConfig instance for this engine (wraps sd.core.ConfigEngine). + debug (bool, optional): + log all bus messages to the debug level + (this can mean a lot of spam, but can also be useful if things are + misbehaving) + blocking (bool, optional): + if set to true, attempts to access the .results property will block + for .queue_timeout seconds waiting for results. If no results are + ready after that, None is returned. If set to false, and a result is + not ready, None will be returned immediately. + + Attributes: + logger (:obj:`logging.Logger`): + Python logger for the class. + queue_timeout (int): + (default: 15 seconds) timeout for the blocking argument/attribute. + feed_name (str): + (default: 'default') the feed name portion of the uri. + web_root (str): + The default web root path. + (default: '/repo/data/web_gui') + IGNORED_MESSAGES(:obj:`tuple` of :obj:`Gst.MessageType`): + Gst.MessageType to be ignored by on_bus_message. + + Examples: + + NOTE: the default GstConfig pipeline is: + uridecodebin ! concat ! identity ... identity ! fakesink, + + Real-world subclasses can override GstConfig to set different source, + sink, and inference elements. See GstConfig documentation for details. + + """ + + IGNORED_MESSAGES = tuple() # type: Tuple[Gst.MessageType] + + logger = logging.getLogger('GstEngine') + # TODO(mdegans): make these properties that warn when a set is attempted + # after the processs has started since these are copied at that point + # (since this is a process) and re-assignment won't work. + queue_timeout=10 + feed_name = 'default' + web_root = '/repo/data/web_gui' + # this is to dump .dot and .pdf + logdir = '/tmp' + + def __init__(self, config:GstConfig, *args, debug=False, blocking=False, **kwargs): + self.logger.debug('__init__') + super().__init__(*args, **kwargs) + # set debug for optional extra logging + self._debug = debug + + # the pipeline configuration + self._gst_config = config # type: GstConfig + + # GStreamer main stuff + self._main_loop = None # type: GLib.MainLoop + self._pipeline = None # type: Gst.Pipeline + # GStreamer elements (in order of connection) + self._sources = [] # type: List[Gst.Element] + self._muxer = None # type: Gst.Element + self._muxer_lock = GLib.Mutex() + self._infer_elements = [] # type: List[Gst.Element] + self._tracker = None # type: Gst.Element + self._distance = None # type: Gst.Element + self._payload = None # type: Gst.Element + self._broker = None # type: Gst.Element + self._osd_converter = None # type: Gst.Element + self._tiler = None # type: Gst.Element + self._tiler_probe_id = None # type: int + self._osd = None # type: Gst.Element + self._sink = None # type: Gst.Element + + # process communication primitives + self._result_queue = multiprocessing.Queue(maxsize=1) + self._stop_requested = multiprocessing.Event() + # todo: make this a property with proper ipc: + # so it can be changed after start + self.blocking=blocking + + @property + def results(self) -> Sequence[str]: + """ + Get results waiting in the queue. + + (may block, depending on self.queue_timeout) + + May return None if no result ready. + + Logs to WARNING level on failure to fetch result. + """ + try: + return self._result_queue.get(block=self.blocking, timeout=self.queue_timeout) + except queue.Empty: + self.logger.warning("failed to get results from queue (queue.Empty)") + return None + except TimeoutError: + self.logger.info("waiting for results...") + return None + + def _update_result_queue(self, results: str): + """ + Called internally by the GStreamer process. + + Update results queue with serialize payload. Should probably be called + by the subclass implemetation of on_buffer(). + + Does not block (because this would block the GLib.MainLoop). + + Can fail if the queue is full in which case the results will + be dropped and logged to the WARNING level. + + Returns: + bool: False on failure, True on success. + """ + if self._result_queue.empty(): + try: + self._result_queue.put_nowait(results) + return True + except queue.Full: + self.logger.warning({'dropped': results}) + return False + + def on_bus_message(self, bus: Gst.Bus, message: Gst.Message, *_) -> bool: + """ + Default bus message callback. + + This implementation does the following on each message type: + + Ignored: + any Gst.MessageType in GstEngine.IGNORED_MESSAGES + + Logged: + Gst.MessageType.STREAM_STATUS + Gst.MessageType.STATE_CHANGED + Gst.MessageType.WARNING + (all others) + + call self._quit(): + Gst.MessageType.EOS + Gst.MessageType.ERROR + """ + # TAG and DURATION_CHANGED seem to be the most common + if message.type in self.IGNORED_MESSAGES: + pass + elif message.type == Gst.MessageType.STREAM_STATUS: + status, owner = message.parse_stream_status() # type: Gst.StreamStatusType, Gst.Element + self.logger.debug(f"{owner.name}:status:{status.value_name}") + elif message.type == Gst.MessageType.STATE_CHANGED: + old, new, _ = message.parse_state_changed() # type: Gst.State, Gst.State, Gst.State + self.logger.debug( + f"{message.src.name}:state-change:" + f"{old.value_name}->{new.value_name}") + elif message.type == Gst.MessageType.EOS: + self.logger.debug(f"Got EOS") + self._quit() + elif message.type == Gst.MessageType.ERROR: + err, errmsg = message.parse_error() # type: GLib.Error, str + self.logger.error(f'{err}: {errmsg}') + self._quit() + elif message.type == Gst.MessageType.WARNING: + err, errmsg = message.parse_warning() # type: GLib.Error, str + self.logger.warning(f'{err}: {errmsg}') + else: + if self._debug: + self.logger.debug( + f"{message.src.name}:{Gst.MessageType.get_name(message.type)}") + return True + + def _create_pipeline(self) -> bool: + """ + Attempt to create pipeline bin. + + Returns: + bool: False on failure, True on success. + """ + # create the pipeline and check + self.logger.debug('creating pipeline') + self._pipeline = Gst.Pipeline() + if not self._pipeline: + self.logger.error('could not create Gst.Pipeline element') + return False + return True + + # TODO(mdegans): some of these creation methods can probably be combined + + def _create_sources(self) -> bool: + # create a source and check + for conf in self._gst_config.src_configs: + self.logger.debug(f'creating source: {self._gst_config.SRC_TYPE}') + src = Gst.ElementFactory.make(self._gst_config.SRC_TYPE) # type: Gst.Element + if not src: + self.logger.error(f'could not create source of type: {self._gst_config.SRC_TYPE}') + return False + + self.logger.debug('') + + # set properties on the source + for k, v in conf.items(): + src.set_property(k, v) + src.set_property('async_handling', True) + src.set_property('caps', Gst.Caps.from_string("video/x-raw(ANY)")) + src.set_property('expose-all-streams', False) + + + # add the source to the pipeline and check + self._pipeline.add(src) + + # append the source to the _sources list + self._sources.append(src) + return True + _create_sources.__doc__ = _ELEM_DOC.format(elem_name='`self.config.SRC_TYPE`') + + def _create_element(self, e_type:str) -> Optional[Gst.Element]: + """ + Create a Gst.Element and add to the pipeline. + + Arguments: + e_type (str): + The FOO_TYPE of elememt to add defined on the config class + as an attribute eg. MUXER_TYPE, SRC_TYPE... This argument is + case insensitive. choices are: ('muxer', 'src', 'sink') + + Once the element of the corresponding type on the config is + made using Gst.ElementFactory.make, it will be added to + self._pipeline and assigned to self._e_type. + + Returns: + A Gst.Element if sucessful, otherwise None. + + Raises: + AttributeError if e_type doesn't exist on the config and the class. + """ + # NOTE(mdegans): "type" and "name" are confusing variable names considering + # GStreamer's and Python's usage of them. Synonyms anybody? + e_type = e_type.lower() + e_name = getattr(self._gst_config, f'{e_type.upper()}_TYPE') + props = getattr(self._gst_config, f'{e_type}_config') # type: dict + self.logger.debug(f'creating {e_type}: {e_name} with props: {props}') + + # make an self.gst_config.E_TYPE_TYPE element + elem = Gst.ElementFactory.make(e_name) + if not elem: + self.logger.error(f'could not create {e_type}: {e_name}') + return + + # set properties on the element + if props: + for k, v in props.items(): + elem.set_property(k, v) + + # assign the element to self._e_type + setattr(self, f'_{e_type}', elem) + + # add the element to the pipeline and check + self._pipeline.add(elem) + + return elem + + def _create_infer_elements(self) -> bool: + """ + Create GstConfig.INFER_TYPE elements, add them to the pipeline, + and append them to self._infer_elements for ease of access / linking. + + Returns: + bool: False on failure, True on success. + """ + self.logger.debug('creating inference elements') + for conf in self._gst_config.infer_configs: + # create and check inference element + elem = Gst.ElementFactory.make(self._gst_config.INFER_TYPE) # type: Gst.Element + if not elem: + self.logger.error(f"failed to create {self._gst_config.INFER_TYPE} element") + return False + + # set properties on inference element + for k, v in conf.items(): + elem.set_property(k, v) + + # add the elements to the pipeline and check + self._pipeline.add(elem) + # oddly, add returns false even when the log shows success + + # append the element to the list of inference elements + self._infer_elements.append(elem) + return True + + def _create_sink(self, pipe_string: str = None): + """ + Create a Gst.Bin sink from a pipeline string + """ + try: + #TODO(mdegans): urlparse and path join on the paths + # (to validate the uri and avoid "//" and such) + public_url = self._gst_config.master_config.config['App']['PublicUrl'] + playlist_root = f'{public_url}/static/gstreamer/{self.feed_name}' + #TODO(mdegans): make the base path a uri for testing + video_root = f'{self.web_root}/static/gstreamer/{self.feed_name}' + if not pipe_string: + encoder = self._gst_config.master_config.config['App']['Encoder'] + pipe_string = f' {encoder} ! mpegtsmux ! hlssink ' \ + f'sync=true ' \ + f'max-files=15 target-duration=5 ' \ + f'playlist-root={playlist_root} ' \ + f'location={video_root}/video_%05d.ts ' \ + f'playlist-location={video_root}/playlist.m3u8' + self.logger.debug(f'sink bin string: {pipe_string}') + self._sink = Gst.parse_bin_from_description(pipe_string, True) + dot_filename = bin_to_pdf( + self._sink, Gst.DebugGraphDetails.ALL, f'{self.__class__.__name__}.sink.created') + if dot_filename: + self.logger.debug( + f'.dot file written to {dot_filename}') + if not self._sink: + # i don't think it's possble to get here unless gstreamer is + # broken + return False + self._pipeline.add(self._sink) + + return True + except (GLib.Error, KeyError): + self.logger.error("sink creation failed", exc_info=True) + return False + + def _create_all(self) -> int: + """ + Create and link the pipeline from self.config. + + Returns: + bool: False on failure, True on success. + """ + create_funcs = ( + self._create_pipeline, + self._create_sources, + functools.partial(self._create_element, 'muxer'), + functools.partial(self._create_element, 'tracker'), + self._create_infer_elements, + functools.partial(self._create_element, 'distance'), + functools.partial(self._create_element, 'payload'), + functools.partial(self._create_element, 'broker'), + functools.partial(self._create_element, 'osd_converter'), + functools.partial(self._create_element, 'tiler'), + functools.partial(self._create_element, 'osd'), + self._create_sink, + ) + + for i, f in enumerate(create_funcs): + if not f(): + self.logger.error( + f"Failed to create DsEngine pipeline at step {i}") + return False + return True + + def _on_source_src_pad_create(self, element: Gst.Element, src_pad: Gst.Pad): + """ + Callback to link sources to the muxer. + """ + # a lock is required so that identical pads are not requested. + # GLib.Mutex is required because python's isn't respected by GLib's MainLoop + self._muxer_lock.lock() + try: + self.logger.debug(f'{element.name} new pad: {src_pad.name}') + self.logger.debug( + f'{src_pad.name} caps:{src_pad.props.caps}') + muxer_sink_pad_name = f'sink_{self._muxer.numsinkpads}' + self.logger.debug(f'{self._muxer.name}:requesting pad:{muxer_sink_pad_name}') + muxer_sink = self._muxer.get_request_pad(muxer_sink_pad_name) + if not muxer_sink: + self.logger.error( + f"failed to get request pad from {self._muxer.name}") + self.logger.debug( + f'{muxer_sink.name} caps:{muxer_sink.props.caps}') + ret = src_pad.link(muxer_sink) + if not ret == Gst.PadLinkReturn.OK: + self.logger.error( + f"failed to link source to muxer becase {ret.value_name}") + self._quit() + finally: + self._muxer_lock.unlock() + + def _link_pipeline(self) -> bool: + """ + Attempt to link the entire pipeline. + + Returns: + bool: False on failure, True on success. + """ + self.logger.debug('linking pipeline') + + # arrange for the sources to link to the muxer when they are ready + # (uridecodebin has "Sometimes" pads so needs to be linked by callback) + for source in self._sources: # type: Gst.Element + source.connect('pad-added', self._on_source_src_pad_create) + + try: + # link the muxer to the first inference element + link(self._muxer, self._infer_elements[0]) + link(self._infer_elements[0], self._tracker) + # if there are secondary inference elements + if self._infer_elements[1:]: + link_many((self._tracker, *self._infer_elements[1:])) + # link the final inference element to distancing engine + link(self._infer_elements[-1], self._distance) + else: + # link tracker directly to the distancing element + link(self._tracker, self._distance) + link(self._distance, self._payload) + # TODO(mdegans): rename osd_converter + # (this requires some changes elsewhere) + link(self._payload, self._osd_converter) + link(self._osd_converter, self._tiler) + link(self._tiler, self._osd) + link(self._osd, self._sink) + except GstLinkError as err: + self.logger.error(f"pipeline link fail because: {err}") + return False + self.logger.debug('linking pipeline successful') + return True + + def on_buffer(self, pad: Gst.Pad, info: Gst.PadProbeInfo, _: None, ) -> Gst.PadProbeReturn: + """ + Default source pad probe buffer callback for the sink. + + Simply returns Gst.PadProbeReturn.OK, signaling the buffer + shuould continue down the pipeline. + """ + return Gst.PadProbeReturn.OK + + def stop(self): + """Stop the GstEngine process.""" + self.logger.info('requesting stop') + self._stop_requested.set() + + def _quit(self) -> Gst.StateChangeReturn: + """ + Quit the GLib.MainLoop and set the pipeline to NULL. + + Called by _on_stop. A separate function for testing purposes. + """ + self.logger.info(f'{self.__class__.__name__} quitting.') + if self._main_loop and self._main_loop.is_running(): + self._main_loop.quit() + if self._pipeline: + self._write_pdf('quit') + self.logger.debug('shifting pipeline to NULL state') + ret = self._pipeline.set_state(Gst.State.NULL) + if ret == Gst.StateChangeReturn.ASYNC: + ret = self._pipeline.get_state(10) + if ret == Gst.StateChangeReturn.SUCCESS: + return + else: + self.logger.error( + 'Failed to quit cleanly. Self terminating.') + self.terminate() # send SIGINT to self + + def _on_stop(self): + """ + Callback to shut down the process if stop() has been called. + """ + if self._stop_requested.is_set(): + self.logger.info(f'stopping {self.__class__.__name__}') + self._quit() + # clear stop_requested state + self._stop_requested.clear() + self.logger.info(f'{self.__class__.__name__} cleanly stopped') + + def _write_pdf(self, suffix: str): + # create a debug pdf from the pipeline + dot_filename = bin_to_pdf( + self._pipeline, Gst.DebugGraphDetails.ALL, f'{self.__class__.__name__}.pipeline.{suffix}') + if dot_filename: + self.logger.debug( + f'.dot file written to {dot_filename}') + + def run(self): + """Called on start(). Do not call this directly.""" + self.logger.debug('run() called. Initializing Gstreamer.') + + # set the .dot file dump path (this must be done prior to Gst.init) + if 'GST_DEBUG_DUMP_DOT_DIR' not in os.environ: + os.environ['GST_DEBUG_DUMP_DOT_DIR'] = self.logdir + + # initialize GStreamer + Gst.init_check(None) + + # create pipeline, + # create and add all elements: + if not self._create_all(): + self.logger.debug('could not create pipeline') + return self._quit() + + # register bus message callback + bus = self._pipeline.get_bus() + if not bus: + self.logger.error('could not get bus') + return self._quit() + + self.logger.debug('registering bus message callback') + bus.add_watch(GLib.PRIORITY_DEFAULT, self.on_bus_message, None) + + # link all pipeline elements: + if not self._link_pipeline(): + self.logger.error('could not link pipeline') + return self._quit() + + # register pad probe buffer callback on the tiler + self.logger.debug('registering self.on_buffer() callback on osd sink pad') + tiler_sink_pad = self._tiler.get_static_pad('sink') + if not tiler_sink_pad: + self.logger.error('could not get osd sink pad') + return self._quit() + + self._tiler_probe_id = tiler_sink_pad.add_probe( + Gst.PadProbeType.BUFFER, self.on_buffer, None) + + # register callback to check for the stop event when idle. + # TODO(mdegans): test to see if a higher priority is needed. + self.logger.debug('registering self._on_stop() idle callback with GLib MainLoop') + GLib.idle_add(self._on_stop) + + # write a pdf before we attempt to start the pipeline + self._write_pdf('linked') + + # set the pipeline to the playing state + self.logger.debug('setting pipeline to PLAYING state') + self._pipeline.set_state(Gst.State.PLAYING) + + # write a pipeline after set the pipeline to PLAYING + self._write_pdf('playing') + + # create and run the main loop. + # this has a built-in signal handler for SIGINT + self.logger.debug('creating the GLib.MainLoop') + self._main_loop = GLib.MainLoop() + self.logger.debug('starting the GLib.MainLoop') + self._main_loop.run() + self.logger.info("complete.") diff --git a/libs/detectors/deepstream/_pyds.py b/libs/detectors/deepstream/_pyds.py new file mode 100644 index 00000000..f5f603aa --- /dev/null +++ b/libs/detectors/deepstream/_pyds.py @@ -0,0 +1,83 @@ +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Python DeepStream bindings loader. + +This is necessary because Nvidia has no proper python package for the bindings +and a straight-up "import pyds" will not work without sys.path hackery. + +Attributes: + PYDS_PATH (str): + The platform specific path to pyds.so (the bindings). + This path is inserted into sys.path. + PYDS_INSTRUCTIONS (str): + URI for (overly complicated) Installation instructions + for the Python DeepStream bindings. +""" + +import os +import logging +import sys +import platform + +from libs.detectors.deepstream._ds_utils import find_deepstream + +logger = logging.getLogger() + +__all__ = [ + 'PYDS_PATH', + 'PYDS_INSTRUCTIONS', + 'pyds' +] + +# Python DeepStream paths + +DS_INFO = find_deepstream() +if DS_INFO: + DS_ROOT = DS_INFO[1] +else: + raise ImportError( + 'DeepStream not intalled. ' + 'Install with: sudo-apt install deepstream-$VERSION') +PYDS_ROOT = os.path.join(DS_ROOT, 'sources/python/bindings') if DS_ROOT else '/' +PYDS_JETSON_PATH = os.path.join(PYDS_ROOT, 'jetson') +PYDS_x86_64_PATH = os.path.join(PYDS_ROOT, 'x86_64') +# Installing the bindings is actually fairly cumbersome, unfortunately. Please, Nvidia +# put more effort into testing and packaging your products. Speed is not enough. +PYDS_INSTRUCTIONS = 'https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/master/HOWTO.md#running-sample-applications' +if platform.machine() == 'aarch64': + PYDS_PATH = PYDS_JETSON_PATH +elif platform.machine() == 'x86_64': + PYDS_PATH = PYDS_x86_64_PATH +else: + logger.warning( + f"unsupported platform for DeepStream Python bindings") + PYDS_PATH = None + +if PYDS_PATH: + sys.path.insert(0, PYDS_PATH) + try: + import pyds + except ImportError: + logger.warning( + f'pyds could not be imported. ' + f'install instructions: {PYDS_INSTRUCTIONS}') + raise diff --git a/libs/distance_pb2.py b/libs/distance_pb2.py new file mode 100644 index 00000000..9940d73a --- /dev/null +++ b/libs/distance_pb2.py @@ -0,0 +1,259 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: distance.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='distance.proto', + package='distanceproto', + syntax='proto3', + serialized_pb=_b('\n\x0e\x64istance.proto\x12\rdistanceproto\"A\n\x05\x42\x61tch\x12\x12\n\nmax_frames\x18\x01 \x01(\r\x12$\n\x06\x66rames\x18\x02 \x03(\x0b\x32\x14.distanceproto.Frame\"h\n\x05\x46rame\x12\x11\n\tframe_num\x18\x01 \x01(\x05\x12\x11\n\tsource_id\x18\x02 \x01(\r\x12%\n\x06people\x18\x03 \x03(\x0b\x32\x15.distanceproto.Person\x12\x12\n\nsum_danger\x18\x04 \x01(\x02\"_\n\x06Person\x12\x0b\n\x03uid\x18\x01 \x01(\x05\x12\x11\n\tis_danger\x18\x02 \x01(\x08\x12\x12\n\ndanger_val\x18\x03 \x01(\x02\x12!\n\x04\x62\x62ox\x18\x04 \x01(\x0b\x32\x13.distanceproto.BBox\"@\n\x04\x42\x42ox\x12\x0c\n\x04left\x18\x01 \x01(\r\x12\x0b\n\x03top\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\r\n\x05width\x18\x04 \x01(\rb\x06proto3') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_BATCH = _descriptor.Descriptor( + name='Batch', + full_name='distanceproto.Batch', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='max_frames', full_name='distanceproto.Batch.max_frames', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='frames', full_name='distanceproto.Batch.frames', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=33, + serialized_end=98, +) + + +_FRAME = _descriptor.Descriptor( + name='Frame', + full_name='distanceproto.Frame', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='frame_num', full_name='distanceproto.Frame.frame_num', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='source_id', full_name='distanceproto.Frame.source_id', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='people', full_name='distanceproto.Frame.people', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sum_danger', full_name='distanceproto.Frame.sum_danger', index=3, + number=4, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=100, + serialized_end=204, +) + + +_PERSON = _descriptor.Descriptor( + name='Person', + full_name='distanceproto.Person', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='uid', full_name='distanceproto.Person.uid', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='is_danger', full_name='distanceproto.Person.is_danger', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='danger_val', full_name='distanceproto.Person.danger_val', index=2, + number=3, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bbox', full_name='distanceproto.Person.bbox', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=206, + serialized_end=301, +) + + +_BBOX = _descriptor.Descriptor( + name='BBox', + full_name='distanceproto.BBox', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='left', full_name='distanceproto.BBox.left', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='top', full_name='distanceproto.BBox.top', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='height', full_name='distanceproto.BBox.height', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='width', full_name='distanceproto.BBox.width', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=303, + serialized_end=367, +) + +_BATCH.fields_by_name['frames'].message_type = _FRAME +_FRAME.fields_by_name['people'].message_type = _PERSON +_PERSON.fields_by_name['bbox'].message_type = _BBOX +DESCRIPTOR.message_types_by_name['Batch'] = _BATCH +DESCRIPTOR.message_types_by_name['Frame'] = _FRAME +DESCRIPTOR.message_types_by_name['Person'] = _PERSON +DESCRIPTOR.message_types_by_name['BBox'] = _BBOX + +Batch = _reflection.GeneratedProtocolMessageType('Batch', (_message.Message,), dict( + DESCRIPTOR = _BATCH, + __module__ = 'distance_pb2' + # @@protoc_insertion_point(class_scope:distanceproto.Batch) + )) +_sym_db.RegisterMessage(Batch) + +Frame = _reflection.GeneratedProtocolMessageType('Frame', (_message.Message,), dict( + DESCRIPTOR = _FRAME, + __module__ = 'distance_pb2' + # @@protoc_insertion_point(class_scope:distanceproto.Frame) + )) +_sym_db.RegisterMessage(Frame) + +Person = _reflection.GeneratedProtocolMessageType('Person', (_message.Message,), dict( + DESCRIPTOR = _PERSON, + __module__ = 'distance_pb2' + # @@protoc_insertion_point(class_scope:distanceproto.Person) + )) +_sym_db.RegisterMessage(Person) + +BBox = _reflection.GeneratedProtocolMessageType('BBox', (_message.Message,), dict( + DESCRIPTOR = _BBOX, + __module__ = 'distance_pb2' + # @@protoc_insertion_point(class_scope:distanceproto.BBox) + )) +_sym_db.RegisterMessage(BBox) + + +# @@protoc_insertion_point(module_scope) \ No newline at end of file diff --git a/neuralet-distancing.py b/neuralet-distancing.py index 29f220e3..d12030a4 100644 --- a/neuralet-distancing.py +++ b/neuralet-distancing.py @@ -1,62 +1,87 @@ #!/usr/bin/python3 import argparse -from multiprocessing import Process +import logging +import sys import threading + +from multiprocessing import Process from libs.config_engine import ConfigEngine -import logging + logger = logging.getLogger(__name__) -def start_engine(config, video_path): - import pdb, traceback, sys +def start_cv_engine(config, video_path): try: if video_path: from libs.core import Distancing as CvEngine engine = CvEngine(config) engine.process_video(video_path) else: - logger.info('Skipping CVEngine as video_path is not set') - except: - extype, value, tb = sys.exc_info() - traceback.print_exc() - pdb.post_mortem(tb) + logger.error('"VideoPath" not set in .ini [App] section') + except Exception: + # this runs sys.excinfo() and logs the result + logger.error("CvEngine failed.", exc_info=True) def start_web_gui(config): from ui.web_gui import WebGUI ui = WebGUI(config) ui.start() - -def main(config): - logging.basicConfig(level=logging.INFO) +def main(config, verbose=False): + logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) if isinstance(config, str): config = ConfigEngine(config) - video_path = config.get_section_dict("App").get("VideoPath", None) - process_engine = Process(target=start_engine, args=(config, video_path,)) + + # create our inference process + try: + # try to launch deepstream but skip if any import errors + # (eg. pyds not found, gi not found) + from libs.detectors.deepstream import DsEngine, DsConfig + process_engine = DsEngine(DsConfig(config)) + except ImportError: + # DeepStream is not available. Let's try CvEngine + process_engine = Process(target=start_cv_engine, args=(config, video_path,)) + + # create our ui process process_api = Process(target=start_web_gui, args=(config,)) + # start both processes process_api.start() process_engine.start() logger.info("Services Started.") + # wait forever forever = threading.Event() try: forever.wait() except KeyboardInterrupt: logger.info("Received interrupt. Terminating...") - process_engine.terminate() - process_engine.join() - logger.info("CV Engine terminated.") + if hasattr(process_engine, 'stop'): + # DsEngine shuts down by asking + # GLib.MainLoop to quit. SIGTERM does this too, + # but it wouldn't call some extra debug stuff + # that's in GstEngine's quit() + # (debug .dot file, .pdf if graphviz is available) + # .stop() will call .terminate() if it times out. + process_engine.stop() + process_engine.join() + else: + process_engine.terminate() + process_engine.join() + + logger.info("Inference Engine terminated.") process_api.terminate() process_api.join() logger.info("Web GUI terminated.") + return 0 if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--config', required=True) + parser.add_argument('--verbose', action='store_true') args = parser.parse_args() - main(args.config) + sys.exit(main(args.config, args.verbose)) diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..2455770d --- /dev/null +++ b/requirements.in @@ -0,0 +1,8 @@ +aiofiles +fastapi +numpy +Pillow +protobuf +scipy +setuptools +uvicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..44385535 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,200 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes +# +aiofiles==0.5.0 \ + --hash=sha256:377fdf7815cc611870c59cbd07b68b180841d2a2b79812d8c218be02448c2acb \ + --hash=sha256:98e6bcfd1b50f97db4980e182ddd509b7cc35909e903a8fe50d8849e02d815af \ + # via -r requirements.in +click==7.1.2 \ + --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ + --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \ + # via uvicorn +dataclasses==0.7 \ + --hash=sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836 \ + --hash=sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6 \ + # via pydantic +fastapi==0.55.1 \ + --hash=sha256:912bc1a1b187146fd74dd45e17ea10aede3d962c921142c412458e911c50dc4c \ + --hash=sha256:b1a96ea772f10cd0235eb09d6e282b1f5e6135dad5121ed80d6beb8fa932e075 \ + # via -r requirements.in +h11==0.9.0 \ + --hash=sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1 \ + --hash=sha256:4bc6d6a1238b7615b266ada57e0618568066f57dd6fa967d1290ec9309b2f2f1 \ + # via uvicorn +httptools==0.1.1 \ + --hash=sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be \ + --hash=sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d \ + --hash=sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce \ + --hash=sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2 \ + --hash=sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6 \ + --hash=sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f \ + --hash=sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009 \ + --hash=sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce \ + --hash=sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a \ + --hash=sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c \ + --hash=sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4 \ + --hash=sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437 \ + # via uvicorn +numpy==1.18.3 \ + --hash=sha256:0aa2b318cf81eb1693fcfcbb8007e95e231d7e1aa24288137f3b19905736c3ee \ + --hash=sha256:163c78c04f47f26ca1b21068cea25ed7c5ecafe5f5ab2ea4895656a750582b56 \ + --hash=sha256:1e37626bcb8895c4b3873fcfd54e9bfc5ffec8d0f525651d6985fcc5c6b6003c \ + --hash=sha256:264fd15590b3f02a1fbc095e7e1f37cdac698ff3829e12ffdcffdce3772f9d44 \ + --hash=sha256:3d9e1554cd9b5999070c467b18e5ae3ebd7369f02706a8850816f576a954295f \ + --hash=sha256:40c24960cd5cec55222963f255858a1c47c6fa50a65a5b03fd7de75e3700eaaa \ + --hash=sha256:46f404314dbec78cb342904f9596f25f9b16e7cf304030f1339e553c8e77f51c \ + --hash=sha256:4847f0c993298b82fad809ea2916d857d0073dc17b0510fbbced663b3265929d \ + --hash=sha256:48e15612a8357393d176638c8f68a19273676877caea983f8baf188bad430379 \ + --hash=sha256:6725d2797c65598778409aba8cd67077bb089d5b7d3d87c2719b206dc84ec05e \ + --hash=sha256:99f0ba97e369f02a21bb95faa3a0de55991fd5f0ece2e30a9e2eaebeac238921 \ + --hash=sha256:a41f303b3f9157a31ce7203e3ca757a0c40c96669e72d9b6ee1bce8507638970 \ + --hash=sha256:a4305564e93f5c4584f6758149fd446df39fd1e0a8c89ca0deb3cce56106a027 \ + --hash=sha256:a551d8cc267c634774830086da42e4ba157fa41dd3b93982bc9501b284b0c689 \ + --hash=sha256:a6bc9432c2640b008d5f29bad737714eb3e14bb8854878eacf3d7955c4e91c36 \ + --hash=sha256:c60175d011a2e551a2f74c84e21e7c982489b96b6a5e4b030ecdeacf2914da68 \ + --hash=sha256:e46e2384209c91996d5ec16744234d1c906ab79a701ce1a26155c9ec890b8dc8 \ + --hash=sha256:e607b8cdc2ae5d5a63cd1bec30a15b5ed583ac6a39f04b7ba0f03fcfbf29c05b \ + --hash=sha256:e94a39d5c40fffe7696009dbd11bc14a349b377e03a384ed011e03d698787dd3 \ + --hash=sha256:eb2286249ebfe8fcb5b425e5ec77e4736d53ee56d3ad296f8947f67150f495e3 \ + --hash=sha256:fdee7540d12519865b423af411bd60ddb513d2eb2cd921149b732854995bbf8b \ + # via -r requirements.in, scipy +pillow==7.1.2 \ + --hash=sha256:04766c4930c174b46fd72d450674612ab44cca977ebbcc2dde722c6933290107 \ + --hash=sha256:0e2a3bceb0fd4e0cb17192ae506d5f082b309ffe5fc370a5667959c9b2f85fa3 \ + --hash=sha256:0f01e63c34f0e1e2580cc0b24e86a5ccbbfa8830909a52ee17624c4193224cd9 \ + --hash=sha256:12e4bad6bddd8546a2f9771485c7e3d2b546b458ae8ff79621214119ac244523 \ + --hash=sha256:1f694e28c169655c50bb89a3fa07f3b854d71eb47f50783621de813979ba87f3 \ + --hash=sha256:3d25dd8d688f7318dca6d8cd4f962a360ee40346c15893ae3b95c061cdbc4079 \ + --hash=sha256:4b02b9c27fad2054932e89f39703646d0c543f21d3cc5b8e05434215121c28cd \ + --hash=sha256:9744350687459234867cbebfe9df8f35ef9e1538f3e729adbd8fde0761adb705 \ + --hash=sha256:a0b49960110bc6ff5fead46013bcb8825d101026d466f3a4de3476defe0fb0dd \ + --hash=sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3 \ + --hash=sha256:b37bb3bd35edf53125b0ff257822afa6962649995cbdfde2791ddb62b239f891 \ + --hash=sha256:b532bcc2f008e96fd9241177ec580829dee817b090532f43e54074ecffdcd97f \ + --hash=sha256:b67a6c47ed963c709ed24566daa3f95a18f07d3831334da570c71da53d97d088 \ + --hash=sha256:b943e71c2065ade6fef223358e56c167fc6ce31c50bc7a02dd5c17ee4338e8ac \ + --hash=sha256:ccc9ad2460eb5bee5642eaf75a0438d7f8887d484490d5117b98edd7f33118b7 \ + --hash=sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d \ + --hash=sha256:eaa83729eab9c60884f362ada982d3a06beaa6cc8b084cf9f76cae7739481dfa \ + --hash=sha256:ee94fce8d003ac9fd206496f2707efe9eadcb278d94c271f129ab36aa7181344 \ + --hash=sha256:f455efb7a98557412dc6f8e463c1faf1f1911ec2432059fa3e582b6000fc90e2 \ + --hash=sha256:f46e0e024346e1474083c729d50de909974237c72daca05393ee32389dabe457 \ + --hash=sha256:f54be399340aa602066adb63a86a6a5d4f395adfdd9da2b9a0162ea808c7b276 \ + --hash=sha256:f784aad988f12c80aacfa5b381ec21fd3f38f851720f652b9f33facc5101cf4d \ + # via -r requirements.in +protobuf==3.12.1 \ + --hash=sha256:04d0b2bd99050d09393875a5a25fd12337b17f3ac2e29c0c1b8e65b277cbfe72 \ + --hash=sha256:05288e44638e91498f13127a3699a6528dec6f9d3084d60959d721bfb9ea5b98 \ + --hash=sha256:175d85370947f89e33b3da93f4ccdda3f326bebe3e599df5915ceb7f804cd9df \ + --hash=sha256:440a8c77531b3652f24999b249256ed01fd44c498ab0973843066681bd276685 \ + --hash=sha256:49fb6fab19cd3f30fa0e976eeedcbf2558e9061e5fa65b4fe51ded1f4002e04d \ + --hash=sha256:4c7cae1f56056a4a2a2e3b00b26ab8550eae738bd9548f4ea0c2fcb88ed76ae5 \ + --hash=sha256:519abfacbb421c3591d26e8daf7a4957763428db7267f7207e3693e29f6978db \ + --hash=sha256:60f32af25620abc4d7928d8197f9f25d49d558c5959aa1e08c686f974ac0b71a \ + --hash=sha256:613ac49f6db266fba243daf60fb32af107cfe3678e5c003bb40a381b6786389d \ + --hash=sha256:954bb14816edd24e746ba1a6b2d48c43576393bbde2fb8e1e3bd6d4504c7feac \ + --hash=sha256:9b1462c033a2cee7f4e8eb396905c69de2c532c3b835ff8f71f8e5fb77c38023 \ + --hash=sha256:c0767f4d93ce4288475afe0571663c78870924f1f8881efd5406c10f070c75e4 \ + --hash=sha256:c45f5980ce32879391144b5766120fd7b8803129f127ce36bd060dd38824801f \ + --hash=sha256:eeb7502f59e889a88bcb59f299493e215d1864f3d75335ea04a413004eb4fe24 \ + --hash=sha256:fdb1742f883ee4662e39fcc5916f2725fec36a5191a52123fec60f8c53b70495 \ + --hash=sha256:fe554066c4962c2db0a1d4752655223eb948d2bfa0fb1c4a7f2c00ec07324f1c \ + # via -r requirements.in +pydantic==1.5.1 \ + --hash=sha256:0a1cdf24e567d42dc762d3fed399bd211a13db2e8462af9dfa93b34c41648efb \ + --hash=sha256:2007eb062ed0e57875ce8ead12760a6e44bf5836e6a1a7ea81d71eeecf3ede0f \ + --hash=sha256:20a15a303ce1e4d831b4e79c17a4a29cb6740b12524f5bba3ea363bff65732bc \ + --hash=sha256:2a6904e9f18dea58f76f16b95cba6a2f20b72d787abd84ecd67ebc526e61dce6 \ + --hash=sha256:3714a4056f5bdbecf3a41e0706ec9b228c9513eee2ad884dc2c568c4dfa540e9 \ + --hash=sha256:473101121b1bd454c8effc9fe66d54812fdc128184d9015c5aaa0d4e58a6d338 \ + --hash=sha256:68dece67bff2b3a5cc188258e46b49f676a722304f1c6148ae08e9291e284d98 \ + --hash=sha256:70f27d2f0268f490fe3de0a9b6fca7b7492b8fd6623f9fecd25b221ebee385e3 \ + --hash=sha256:8433dbb87246c0f562af75d00fa80155b74e4f6924b0db6a2078a3cd2f11c6c4 \ + --hash=sha256:8be325fc9da897029ee48d1b5e40df817d97fe969f3ac3fd2434ba7e198c55d5 \ + --hash=sha256:93b9f265329d9827f39f0fca68f5d72cc8321881cdc519a1304fa73b9f8a75bd \ + --hash=sha256:9be755919258d5d168aeffbe913ed6e8bd562e018df7724b68cabdee3371e331 \ + --hash=sha256:ab863853cb502480b118187d670f753be65ec144e1654924bec33d63bc8b3ce2 \ + --hash=sha256:b96ce81c4b5ca62ab81181212edfd057beaa41411cd9700fbcb48a6ba6564b4e \ + --hash=sha256:da8099fca5ee339d5572cfa8af12cf0856ae993406f0b1eb9bb38c8a660e7416 \ + --hash=sha256:e2c753d355126ddd1eefeb167fa61c7037ecd30b98e7ebecdc0d1da463b4ea09 \ + --hash=sha256:f0018613c7a0d19df3240c2a913849786f21b6539b9f23d85ce4067489dfacfa \ + # via fastapi +scipy==1.4.1 \ + --hash=sha256:00af72998a46c25bdb5824d2b729e7dabec0c765f9deb0b504f928591f5ff9d4 \ + --hash=sha256:0902a620a381f101e184a958459b36d3ee50f5effd186db76e131cbefcbb96f7 \ + --hash=sha256:1e3190466d669d658233e8a583b854f6386dd62d655539b77b3fa25bfb2abb70 \ + --hash=sha256:2cce3f9847a1a51019e8c5b47620da93950e58ebc611f13e0d11f4980ca5fecb \ + --hash=sha256:3092857f36b690a321a662fe5496cb816a7f4eecd875e1d36793d92d3f884073 \ + --hash=sha256:386086e2972ed2db17cebf88610aab7d7f6e2c0ca30042dc9a89cf18dcc363fa \ + --hash=sha256:71eb180f22c49066f25d6df16f8709f215723317cc951d99e54dc88020ea57be \ + --hash=sha256:770254a280d741dd3436919d47e35712fb081a6ff8bafc0f319382b954b77802 \ + --hash=sha256:787cc50cab3020a865640aba3485e9fbd161d4d3b0d03a967df1a2881320512d \ + --hash=sha256:8a07760d5c7f3a92e440ad3aedcc98891e915ce857664282ae3c0220f3301eb6 \ + --hash=sha256:8d3bc3993b8e4be7eade6dcc6fd59a412d96d3a33fa42b0fa45dc9e24495ede9 \ + --hash=sha256:9508a7c628a165c2c835f2497837bf6ac80eb25291055f56c129df3c943cbaf8 \ + --hash=sha256:a144811318853a23d32a07bc7fd5561ff0cac5da643d96ed94a4ffe967d89672 \ + --hash=sha256:a1aae70d52d0b074d8121333bc807a485f9f1e6a69742010b33780df2e60cfe0 \ + --hash=sha256:a2d6df9eb074af7f08866598e4ef068a2b310d98f87dc23bd1b90ec7bdcec802 \ + --hash=sha256:bb517872058a1f087c4528e7429b4a44533a902644987e7b2fe35ecc223bc408 \ + --hash=sha256:c5cac0c0387272ee0e789e94a570ac51deb01c796b37fb2aad1fb13f85e2f97d \ + --hash=sha256:cc971a82ea1170e677443108703a2ec9ff0f70752258d0e9f5433d00dda01f59 \ + --hash=sha256:dba8306f6da99e37ea08c08fef6e274b5bf8567bb094d1dbe86a20e532aca088 \ + --hash=sha256:dc60bb302f48acf6da8ca4444cfa17d52c63c5415302a9ee77b3b21618090521 \ + --hash=sha256:dee1bbf3a6c8f73b6b218cb28eed8dd13347ea2f87d572ce19b289d6fd3fbc59 \ + # via -r requirements.in +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ + # via protobuf +starlette==0.13.2 \ + --hash=sha256:6169ee78ded501095d1dda7b141a1dc9f9934d37ad23196e180150ace2c6449b \ + --hash=sha256:a9bb130fa7aa736eda8a814b6ceb85ccf7a209ed53843d0d61e246b380afa10f \ + # via fastapi +uvicorn==0.11.5 \ + --hash=sha256:50577d599775dac2301bac8bd5b540d19a9560144143c5bdab13cba92783b6e7 \ + --hash=sha256:596eaa8645b6dbc24d6610e335f8ddf5f925b4c4b86fdc7146abb0bf0da65d17 \ + # via -r requirements.in +uvloop==0.14.0 \ + --hash=sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd \ + --hash=sha256:123ac9c0c7dd71464f58f1b4ee0bbd81285d96cdda8bc3519281b8973e3a461e \ + --hash=sha256:4315d2ec3ca393dd5bc0b0089d23101276778c304d42faff5dc4579cb6caef09 \ + --hash=sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726 \ + --hash=sha256:afd5513c0ae414ec71d24f6f123614a80f3d27ca655a4fcf6cabe50994cc1891 \ + --hash=sha256:b4f591aa4b3fa7f32fb51e2ee9fea1b495eb75b0b3c8d0ca52514ad675ae63f7 \ + --hash=sha256:bcac356d62edd330080aed082e78d4b580ff260a677508718f88016333e2c9c5 \ + --hash=sha256:e7514d7a48c063226b7d06617cbb12a14278d4323a065a8d46a7962686ce2e95 \ + --hash=sha256:f07909cd9fc08c52d294b1570bba92186181ca01fe3dc9ffba68955273dd7362 \ + # via uvicorn +websockets==8.1 \ + --hash=sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5 \ + --hash=sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5 \ + --hash=sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308 \ + --hash=sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb \ + --hash=sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a \ + --hash=sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c \ + --hash=sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170 \ + --hash=sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422 \ + --hash=sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8 \ + --hash=sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485 \ + --hash=sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f \ + --hash=sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8 \ + --hash=sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc \ + --hash=sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779 \ + --hash=sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989 \ + --hash=sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1 \ + --hash=sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092 \ + --hash=sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824 \ + --hash=sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d \ + --hash=sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55 \ + --hash=sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36 \ + --hash=sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b \ + # via uvicorn + +# The following packages are considered to be unsafe in a requirements file: +setuptools==47.1.1 \ + --hash=sha256:145fa62b9d7bb544fce16e9b5a9bf4ab2032d2f758b7cd674af09a92736aff74 \ + --hash=sha256:74f33f44290f95c5c4a7c13ccc9d6d1a16837fe9dce0acf411dd244e7de95143 \ + # via -r requirements.in, protobuf diff --git a/test.Dockerfile b/test.Dockerfile new file mode 100644 index 00000000..3761812c --- /dev/null +++ b/test.Dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:latest + +COPY ds_pybind_v0.9.tbz2 ./ From eab488cbb7a89abbf4e48aeee4e184668c930ca2 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 5 Jun 2020 20:57:20 +0000 Subject: [PATCH 04/25] fix class_id --- deepstream.ini | 2 +- libs/detectors/deepstream/_ds_config.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/deepstream.ini b/deepstream.ini index dd668a8c..d6059cc0 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -21,7 +21,7 @@ Device: DeepStream ; Detector's Name can be "resnet10" or "peoplenet" Name: resnet10 ;ImageSize is not needed since this is included in the deepstream config .ini -ClassID: 0 +ClassID: 2 MinScore: 0.25 ; TODO(mdegans): remove unused sections and keys from this file diff --git a/libs/detectors/deepstream/_ds_config.py b/libs/detectors/deepstream/_ds_config.py index 257c15f5..1b8d603a 100644 --- a/libs/detectors/deepstream/_ds_config.py +++ b/libs/detectors/deepstream/_ds_config.py @@ -289,6 +289,12 @@ def tiler_config(self) -> ElemConfig: 'height': self.out_resolution[1], } + @property + def distance_config(self) -> ElemConfig: + return { + 'class-id': int(self.class_ids) + } + @property def infer_configs(self) -> List[ElemConfig]: """ From 7997930e17e62596717ce94d02b4762d05dfbdca Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 19:49:24 +0000 Subject: [PATCH 05/25] base image changes --- deepstream-x86.Dockerfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/deepstream-x86.Dockerfile b/deepstream-x86.Dockerfile index 7f5ae4db..337768fa 100644 --- a/deepstream-x86.Dockerfile +++ b/deepstream-x86.Dockerfile @@ -18,21 +18,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -FROM registry.hub.docker.com/mdegans/gstcudaplugin:latest +ARG CUDA_PLUGIN_TAG="build me with deepstream_docker_build.sh" +FROM registry.hub.docker.com/mdegans/gstcudaplugin:${CUDA_PLUGIN_TAG} # this can't be downloaded directly because a license needs to be accepted, # (because those who abuse it will care so much about that) and a tarball # extracted. This is very un-fun: # https://developer.nvidia.com/deepstream-getting-started#python_bindings -ARG DS_PYBIND_TBZ2='ds_pybind_v0.9.tbz2' ARG DS_SOURCES_ROOT='/opt/nvidia/deepstream/deepstream/sources' +ARG CONFIG_FILE="deepstream.ini" # copy stuff we need at the start of the build -COPY ${DS_PYBIND_TBZ2} requirements.txt /tmp/ - -# extract and install the python bindings -RUN mkdir -p ${DS_SOURCES_ROOT} \ - && tar -xf /tmp/${DS_PYBIND_TBZ2} -C ${DS_SOURCES_ROOT} +COPY requirements.txt /tmp/ # install pip, install requirements, remove pip and deps RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -70,7 +67,9 @@ COPY data ./data/ # drop all caps to a regular user RUN useradd -md /var/smart_distancing -rUs /bin/false smart_distancing \ - && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/gstreamer + && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/gstreamer/default \ + && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/data/default/objects_log + USER smart_distancing:smart_distancing # copy frontend @@ -79,4 +78,4 @@ COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /sr # entrypoint with deepstream. EXPOSE 8000 ENTRYPOINT [ "/usr/bin/python3", "neuralet-distancing.py" ] -CMD [ "--config", "deepstream.ini" ] +CMD [ "--verbose", "--config", "deepstream.ini" ] From cc53465b0287811b234dfe22782949d0c3a60cdb Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 19:49:56 +0000 Subject: [PATCH 06/25] base image tagging --- deepstream_docker_build.sh | 47 ++++++++++++++--------------- x86.Dockerfile => x86_64.Dockerfile | 0 2 files changed, 22 insertions(+), 25 deletions(-) rename x86.Dockerfile => x86_64.Dockerfile (100%) diff --git a/deepstream_docker_build.sh b/deepstream_docker_build.sh index 60088abb..04f694fb 100755 --- a/deepstream_docker_build.sh +++ b/deepstream_docker_build.sh @@ -24,49 +24,46 @@ set -e # change this to your docker hub user if you fork this and want to push it readonly USER_NAME="neuralet" # DeepStream constants: -readonly DS_PYBIND_TBZ2="ds_pybind_v0.9.tbz2" # deepstrem python bindings readonly DS_PYBIND_URL="https://developer.nvidia.com/deepstream-getting-started#python_bindings" # Dockerfile names -readonly X86_DOCKERFILE="deepstream-x86.Dockerfile" -readonly TEGRA_DOCKERFILE="deepstream-tegra.Dockerfile" +readonly DOCKERFILE="deepstream-$(arch).Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" +# change this to use a newer gst-cuda-plugin version +readonly CUDA_PLUGIN_VER="0.3.1" # get the docker tag suffix from the git branch -# if master, use "latest" -TAG_SUFFIX=$(git rev-parse --abbrev-ref HEAD) -if [[ $TAG_SUFFIX == "master" ]]; then - TAG_SUFFIX="deepstream" +TAG_SUFFIX="deepstream-$(git rev-parse --abbrev-ref HEAD)" +if [[ $TAG_SUFFIX == "deepstream-master" ]]; then + # if we're on master, just use "deepstream" + TAG_SUFFIX="deepstream" fi -function check_deps() { - if [ ! -f ${DS_PYBIND_TBZ2} ]; then - echo "ERROR: ${DS_PYBIND_TBZ2} needed in same directory as Dockerfile." > /dev/stderr - echo "Download from: ${DS_PYBIND_URL}" > /dev/stderr - echo "(it's inside deepstream_python_v0.9.tbz2)" > /dev/stderr - exit 1 - fi -} - function x86() { - exec docker build -f $X86_DOCKERFILE -t "$USER_NAME/smart-distancing:$TAG_SUFFIX" . + exec docker build --pull -f $DOCKERFILE \ + -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ + --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ + . } function tegra() { - exec docker build -f $TEGRA_DOCKERFILE -t "$USER_NAME/smart-distancing:$TAG_SUFFIX" . + exec docker build --pull -f $DOCKERFILE \ + -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ + --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ + . } main() { - check_deps -case "$1" in - x86) - x86 + local ARCH="$(arch)" +case $ARCH in + x86_64) + x86 $ARCH ;; - tegra) - tegra + aarch64) + tegra $ARCH ;; *) - echo "Usage: $THIS_SCRIPT_BASENAME {x86|tegra}" + echo "Unrecognized architecture: $ARCH" esac } diff --git a/x86.Dockerfile b/x86_64.Dockerfile similarity index 100% rename from x86.Dockerfile rename to x86_64.Dockerfile From 9db2d92ecc4a82c2627cd7f5e4d7556ad73d15f2 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:13:22 +0000 Subject: [PATCH 07/25] docker simplification * change for base image * easier build/run script * drop to user at run time --- ...Dockerfile => deepstream-x86_64.Dockerfile | 11 +- deepstream.sh | 104 ++++++++++++++++++ deepstream_docker_build.sh | 16 +-- 3 files changed, 114 insertions(+), 17 deletions(-) rename deepstream-x86.Dockerfile => deepstream-x86_64.Dockerfile (86%) create mode 100755 deepstream.sh mode change 100755 => 100644 deepstream_docker_build.sh diff --git a/deepstream-x86.Dockerfile b/deepstream-x86_64.Dockerfile similarity index 86% rename from deepstream-x86.Dockerfile rename to deepstream-x86_64.Dockerfile index 337768fa..fc9e7fec 100644 --- a/deepstream-x86.Dockerfile +++ b/deepstream-x86_64.Dockerfile @@ -65,17 +65,10 @@ COPY tools ./tools/ COPY logs ./logs/ COPY data ./data/ -# drop all caps to a regular user -RUN useradd -md /var/smart_distancing -rUs /bin/false smart_distancing \ - && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/gstreamer/default \ - && chown -R smart_distancing:smart_distancing /repo/data/web_gui/static/data/default/objects_log - -USER smart_distancing:smart_distancing - # copy frontend COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /srv/frontend # entrypoint with deepstream. EXPOSE 8000 -ENTRYPOINT [ "/usr/bin/python3", "neuralet-distancing.py" ] -CMD [ "--verbose", "--config", "deepstream.ini" ] +ENTRYPOINT [ "/usr/bin/python3", "neuralet-distancing.py", "--config", "deepstream.ini" ] + diff --git a/deepstream.sh b/deepstream.sh new file mode 100755 index 00000000..052dd636 --- /dev/null +++ b/deepstream.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copyright (c) 2020 Michael de Gans +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -e + +# change the default run port +readonly PORT="8000" +# change this to bump the version tag +readonly VERSION="0.1.0" +# change this to your docker hub user if you fork this and want to push it +readonly USER_NAME="neuralet" +# change this to override the arch (should never be necessary) +readonly ARCH="$(arch)" +# Dockerfile name +readonly DOCKERFILE="deepstream-$ARCH.Dockerfile" +# https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ +readonly THIS_SCRIPT_BASENAME="${0##*/}" +# change this to use a newer gst-cuda-plugin version +readonly CUDA_PLUGIN_VER="0.3.1" +# https://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself +readonly THIS_DIR="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" +# the primary group to use +if [[ $ARCH = "aarch64" ]]; then + # on tegra, a user must be in the video group to use the gpu + readonly GROUP_ID="$(cut -d: -f3 < <(getent group video))" + declare readonly GPU_ARGS=( + "--runtime" + "nvidia" + ) +else + readonly GROUP_ID=$(id -g) + declare readonly GPU_ARGS=( + "--gpus" + "all" + ) +fi + +# this helps tag the image +GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +# get the docker tag suffix from the git branch +if [[ $GIT_BRANCH == "master" ]]; then + # if we're on master, just use "deepstream" + readonly GIT_BRANCH="latest" +else + readonly GIT_BRANCH=$GIT_BRANCH +fi + +# change this to ovverride the image tag suffix +readonly TAG_SUFFIX1="deepstream-$VERSION-$ARCH" +readonly TAG_SUFFIX2="deepstream-$GIT_BRANCH-$ARCH" + +function build() { + set -x + exec docker build --pull -f $DOCKERFILE \ + -t "$USER_NAME/smart-distancing:$TAG_SUFFIX1" \ + -t "$USER_NAME/smart-distancing:$TAG_SUFFIX2" \ + --build-arg CUDA_PLUGIN_TAG="$CUDA_PLUGIN_VER-$ARCH" \ + . +} + +function run() { + set -x + exec docker run -it --rm \ + "${GPU_ARGS[@]}" \ + -v "$THIS_DIR/deepstream.ini:/repo/deepstream.ini" \ + -v "$THIS_DIR/data:/repo/data" \ + --user $UID:$GROUP_ID \ + -p "$PORT:8000" \ + "$USER_NAME/smart-distancing:$TAG_SUFFIX1" \ + "$@" +} + +main() { +case "$1" in + build) + build + ;; + run) + run + ;; + *) + echo "Usage: $THIS_SCRIPT_BASENAME {build|run}" +esac +} + +main "$1" \ No newline at end of file diff --git a/deepstream_docker_build.sh b/deepstream_docker_build.sh old mode 100755 new mode 100644 index 04f694fb..5f64886e --- a/deepstream_docker_build.sh +++ b/deepstream_docker_build.sh @@ -39,14 +39,14 @@ if [[ $TAG_SUFFIX == "deepstream-master" ]]; then TAG_SUFFIX="deepstream" fi -function x86() { +function build() { exec docker build --pull -f $DOCKERFILE \ -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ . } -function tegra() { +function run() { exec docker build --pull -f $DOCKERFILE \ -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ @@ -55,15 +55,15 @@ function tegra() { main() { local ARCH="$(arch)" -case $ARCH in - x86_64) - x86 $ARCH +case "$1" in + build) + build $ARCH ;; - aarch64) - tegra $ARCH + run) + run $ARCH ;; *) - echo "Unrecognized architecture: $ARCH" + echo "Usage: $ARCH" esac } From 80059ffa8d22deb94a501c9c14f5516e633cbdc1 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:13:54 +0000 Subject: [PATCH 08/25] log directory change --- deepstream.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.ini b/deepstream.ini index d6059cc0..f7ba2a71 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -38,4 +38,4 @@ DistMethod: CenterPointsDistance ; options: csv, json (default is csv if not set) Name: csv ; optional log path (default to ~/.smart_distancing/logs/): -;LogDirectory: /some/path +LogDirectory: /repo/data/web_gui/static/data From d14dce9ecb8cef2c5533b6caf8d7296adf946b3a Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:14:31 +0000 Subject: [PATCH 09/25] use new plugins --- libs/detectors/deepstream/__init__.py | 17 +- libs/detectors/deepstream/_base_detector.py | 194 ------------------- libs/detectors/deepstream/_detectors.py | 122 ------------ libs/detectors/deepstream/_ds_config.py | 15 +- libs/detectors/deepstream/_ds_engine.py | 203 -------------------- libs/detectors/deepstream/_gst_engine.py | 87 ++------- libs/detectors/deepstream/_pyds.py | 83 -------- neuralet-distancing.py | 11 +- 8 files changed, 33 insertions(+), 699 deletions(-) delete mode 100644 libs/detectors/deepstream/_base_detector.py delete mode 100644 libs/detectors/deepstream/_detectors.py delete mode 100644 libs/detectors/deepstream/_ds_engine.py delete mode 100644 libs/detectors/deepstream/_pyds.py diff --git a/libs/detectors/deepstream/__init__.py b/libs/detectors/deepstream/__init__.py index 85d17892..c44e9e1c 100644 --- a/libs/detectors/deepstream/__init__.py +++ b/libs/detectors/deepstream/__init__.py @@ -31,29 +31,16 @@ Gst, GLib, ) -from libs.detectors.deepstream._base_detector import * from libs.detectors.deepstream._ds_utils import * -from libs.detectors.deepstream._pyds import * from libs.detectors.deepstream._ds_config import * from libs.detectors.deepstream._gst_engine import * -from libs.detectors.deepstream._ds_engine import * -from libs.detectors.deepstream._detectors import * __all__ = [ - 'BaseDetector', # _base_detector.py 'bin_to_pdf', # _ds_utils.py 'DsConfig', # _ds_config.py - 'DsDetector', # _detectors.py - 'DsEngine', # _ds_engine.py 'ElemConfig', # _ds_config.py 'find_deepstream', # _ds_utils.py - 'frame_meta_iterator', # _ds_engine.py 'GstConfig', # _ds_config.py - 'GstEngine', # _ds_engine.py - 'link_many', # _ds_engine.py - 'obj_meta_iterator', # _ds_engine.py - 'OnFrameCallback', # _base_detector.py - 'PYDS_INSTRUCTIONS', # _pyds.py - 'PYDS_PATH', # _pyds.py - 'pyds', # _pyds.py + 'GstEngine', # _gst_engine.py + 'link_many', # _gst_engine.py ] diff --git a/libs/detectors/deepstream/_base_detector.py b/libs/detectors/deepstream/_base_detector.py deleted file mode 100644 index eb43a152..00000000 --- a/libs/detectors/deepstream/_base_detector.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2020 Michael de Gans -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -Contains Detector, the base class for all detectors (in the deepstream branch, anyway). -""" - -import abc -import logging -import os -import sys -import urllib.parse -import urllib.request -import itertools - -from libs.distance_pb2 import ( - BBox, - Frame, - Person, -) -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, -) - -OnFrameCallback = Callable[[Frame], Any] -Callable.__doc__ = """ -Signature an on-frame callback accepting a fra. -""" - -__all__ = ['BaseDetector', 'OnFrameCallback'] - -class BaseDetector(abc.ABC): - """ - A base class for all Detectors. The following should be overridden: - - PLATFORM the model platform (eg. edgetpu, jetson, x86) - DEFAULT_MODEL_FILE with the desired model filename - DEFAULT_MODEL_URL with the url path minus filename of the model - - load_model() to load the model. This is called for you on __init__. - - Something should also call on_frame() with a sequence of sd.Detection - - Arguments: - config (:obj:`sd.core.ConfigEngine`): - the global config class - on_frame (:obj:`OnFrameCallback`): - A callback to call on every frame. - """ - - PLATFORM = None # type: Tuple - DEFAULT_MODEL_FILE = None # type: str - DEFAULT_MODEL_URL = None # type: str - - # this works differently in the deepstream branch - MODEL_DIR = '/repo/data' - - def __init__(self, config, on_frame:OnFrameCallback=None): - # set the config - self.config = config - - # assign the on_frame callback if any - if on_frame is not None: - self.on_frame = on_frame - - # set up a logger on the class - self.logger = logging.getLogger(self.__class__.__name__) - - # download the model if necessary - if not os.path.isfile(self.model_file): - self.logger.info( - f'model does not exist under: "{self.model_path}" ' - f'downloading from "{self.model_url}"') - os.makedirs(self.model_path, mode=0o755, exist_ok=True) - urllib.request.urlretrieve(self.model_url, self.model_file) - - # add a frame counter - self._frame_count = itertools.count() - - # load the model - self.load_model() - - @property - def detector_config(self) -> Dict: - """:return: the 'Detector' section from self.config""" - return self.config.get_section_dict('Detector') - - @property - def name(self) -> str: - """:return: the detector name.""" - return self.detector_config['Name'] - - @property - def model_path(self) -> Optional[str]: - """:return: the folder containing the model.""" - try: - cfg_model_path = self.detector_config['ModelPath'] - if cfg_model_path: # not None and greater than zero in length - return cfg_model_path - except KeyError: - pass - return os.path.join(self.MODEL_DIR, self.PLATFORM) - - @property - def model_file(self) -> Optional[str]: - """:return: the model filename.""" - return os.path.join(self.model_path, self.DEFAULT_MODEL_FILE) - - @property - def model_url(self) -> str: - """:return: a parsed url pointing to a downloadable model""" - # this is done to validate it's at least a valid uri - # TODO(mdegans?): move to config class - return urllib.parse.urlunparse(urllib.parse.urlparse( - self.DEFAULT_MODEL_URL + self.DEFAULT_MODEL_FILE)) - - @property - def class_id(self) -> int: - """:return: the class id to detect.""" - return int(self.detector_config['ClassID']) - - @property - def score_threshold(self) -> float: - """:return: the detection minimum threshold (MinScore).""" - return float(self.detector_config['MinScore']) - min_score = score_threshold # an alias for ease of access - - @property - @abc.abstractmethod - def sources(self) -> List[str]: - """:return: the active sources.""" - - @sources.setter - @abc.abstractmethod - def sources(self, source: Sequence[str]): - """Set the active sources""" - - @property - @abc.abstractmethod - def fps(self) -> int: - """:return: the current fps""" - - @abc.abstractmethod - def load_model(self): - """load the model. Called by default implementation of __init__.""" - - @abc.abstractmethod - def start(self): - """ - Start the detector (should do inferences and call on_frame). - """ - pass - - @abc.abstractmethod - def stop(self): - """ - Start the detector (should do inferences and call on_frame). - """ - pass - - def on_frame(self, frame: Frame): # pylint: disable=method-hidden - """ - Calculate distances between detections and updates UI. - This default implementation just logs serialized frames to the DEBUG - level and is called if on_frame is not specified on __init__. - - Arguments: - frame (:obj:`Frame`): frame level deserialized protobuf metadata. - """ - self.logger.debug({'frame_proto': frame.SerializeToString()}) - pass diff --git a/libs/detectors/deepstream/_detectors.py b/libs/detectors/deepstream/_detectors.py deleted file mode 100644 index 3a053223..00000000 --- a/libs/detectors/deepstream/_detectors.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2020 Michael de Gans -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -DsDetector lives here -""" - -import logging -import itertools -import time - -from libs.config_engine import ConfigEngine -from libs.detectors.deepstream import ( - BaseDetector, - DsConfig, - DsEngine, - OnFrameCallback, -) - -from libs.distance_pb2 import ( - Batch, - Frame, -) - -from typing import ( - Dict, - Tuple, - Sequence, -) - -__all__ = ['DsDetector'] - -class DsDetector(BaseDetector): - """ - DeepStream implementation of BaseDetector. - """ - - DEFAULT_MODEL_FILE = 'None' - DEFAULT_MODEL_URL = 'None' - - engine = None # type: DsEngine - - def load_model(self): - """ - init/reinit a DsEngine instance (terminates if necessary). - - Called by start() automatically. - """ - if self.engine and self.engine.is_alive(): - self.logger.info( - "restarting engine") - self.engine.terminate() - self.engine.join() - self.engine = DsEngine(DsConfig(self.config)) - - # @Hossein I know the other classes don't have this, but it may make sense - # to add this start + stop functionality to the base class. - def start(self, blocking=True, timeout=10): - """ - Start DsDetector's engine. - - Arguments: - blocking (bool): - Whether to block this thread while waiting for results. If - false, busy waits with a sleep(0) in the loop. - (set False if you want this to spin) - timeout: - If blocking is True, - """ - self.logger.info( - f'starting up{" in blocking mode" if blocking else ""}') - self.engine.blocking=blocking - self.engine.start() - self.engine.queue_timeout=10 - batch = Batch() - while self.engine.is_alive(): - batch_str = self.engine.results - if not batch_str: - time.sleep(0) # this is to switch context if launched in thread - continue - batch.ParseFromString(batch_str) - for frame in batch.frames: # pylint: disable=no-member - next(self._frame_count) - self.on_frame(frame) - - def stop(self): - self.engine.stop() - - @property - def fps(self): - self.logger.warning("fps reporting not yet implemented") - return 30 - - @property - def sources(self): - self.logger.warning("getting sources at runtime not yet implemented") - return [] - - @sources.setter - def sources(self, sources: Sequence[str]): - self.logger.warning("setting sources at runtime not yet implemented") - -if __name__ == "__main__": - import doctest - doctest.testmod() diff --git a/libs/detectors/deepstream/_ds_config.py b/libs/detectors/deepstream/_ds_config.py index 1b8d603a..30565a6d 100644 --- a/libs/detectors/deepstream/_ds_config.py +++ b/libs/detectors/deepstream/_ds_config.py @@ -24,6 +24,7 @@ import os import logging +import datetime from math import ( log, @@ -99,7 +100,6 @@ class GstConfig(object): MUXER_TYPE = 'concat' # using this just because it has request pads INFER_TYPE = 'identity' DISTANCE_TYPE = 'identity' - PAYLOAD_TYPE = 'identity' BROKER_TYPE = 'identity' OSD_CONVERTER_TYPE = 'identity' TILER_TYPE = 'identity' @@ -162,7 +162,6 @@ def _blank_config(self) -> ElemConfig: osd_converter_config = property(_blank_config) sink_config = property(_blank_config) distance_config = property(_blank_config) - payload_config = property(_blank_config) broker_config = property(_blank_config) @property @@ -240,7 +239,6 @@ class DsConfig(GstConfig): MUXER_TYPE = 'nvstreammux' INFER_TYPE = 'nvinfer' DISTANCE_TYPE = 'dsdistance' - PAYLOAD_TYPE = 'dsprotopayload' BROKER_TYPE = 'payloadbroker' OSD_CONVERTER_TYPE = 'nvvideoconvert' TILER_TYPE = 'nvmultistreamtiler' @@ -295,6 +293,17 @@ def distance_config(self) -> ElemConfig: 'class-id': int(self.class_ids) } + @property + def broker_config(self) -> ElemConfig: + return { + 'mode': 2, # csv + 'basepath': os.path.join( + self.master_config.config['Logger']['LogDirectory'], + 'default', 'objects_log', + datetime.datetime.today().strftime('%Y-%m-%d'), + ) + } + @property def infer_configs(self) -> List[ElemConfig]: """ diff --git a/libs/detectors/deepstream/_ds_engine.py b/libs/detectors/deepstream/_ds_engine.py deleted file mode 100644 index 01dbfcf0..00000000 --- a/libs/detectors/deepstream/_ds_engine.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) 2020 Michael de Gans -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -DsEngine lives here (GstEngine multprocessing.Process subclass) -""" - -import configparser -import tempfile -import logging -import queue - -# import gstreamer bidings -import gi -gi.require_version('Gst', '1.0') -gi.require_version('GLib', '2.0') -from gi.repository import ( - Gst, - GLib, -) -# import python deepstream -from libs.detectors.deepstream import pyds -# import config stuff -from libs.detectors.deepstream._ds_config import ( - GstConfig, - DsConfig, - ElemConfig, -) -# import metadata stuff -from libs.distance_pb2 import ( - Batch, - Frame, - Person, - BBox, -) -from libs.detectors.deepstream._gst_engine import GstEngine -# typing -from typing import ( - Any, - Callable, - Iterator, - Iterable, - Optional, - List, - Mapping, - TYPE_CHECKING, -) - -__all__ = [ - 'DsEngine', - 'frame_meta_iterator', - 'obj_meta_iterator', -] - -# these two functions below are used by DsEngine to parse pyds metadata - -def frame_meta_iterator(frame_meta_list: GLib.List, - reverse=False) -> Iterator[pyds.NvDsFrameMeta]: - """ - Iterate through DeepStream frame metadata GList (doubly linked list). - - Arguments: - Reverse (bool): iterate in reverse (with .previous) - """ - # generators catch StopIteration to stop iteration, - while frame_meta_list is not None: - yield pyds.glist_get_nvds_frame_meta(frame_meta_list.data) - # a Glib.List is a doubly linked list where .data is the content - # and 'next' and 'previous' contain to the next and previous elements - frame_meta_list = frame_meta_list.next if not reverse else frame_meta_list.previous - -def obj_meta_iterator(obj_meta_list: GLib.List, - reverse=False) -> Iterator[pyds.NvDsObjectMeta]: - """ - Iterate through DeepStream object metadata GList (doubly linked list). - - Arguments: - Reverse (bool): iterate in reverse (with .previous) - """ - while obj_meta_list is not None: - yield pyds.glist_get_nvds_object_meta(obj_meta_list.data) - obj_meta_list = obj_meta_list.next if not reverse else obj_meta_list.previous - -def write_config(tmpdir, config:dict) -> str: - """ - Write a nvinfer config to a .ini file in tmpdir and return the filename. - - The section heading is [property] - - Example: - >>> config = { - ... 'model-file': 'foo.caffemodel', - ... 'proto-file': 'foo.prototxt', - ... 'labelfile-path': 'foo.labels.txt', - ... 'int8-calib-file': 'foo_cal_trt.bin', - ... } - >>> with tempfile.TemporaryDirectory() as tmp: - ... filename = write_config(tmp, config) - ... print(filename) - ... with open(filename) as f: - ... for l in f: - ... print(l, end='') - /tmp/tmp.../config....ini - [property] - model-file = foo.caffemodel - proto-file = foo.prototxt - labelfile-path = foo.labels.txt - int8-calib-file = foo_cal_trt.bin - - """ - # TODO(mdegans): simple property validation to fail fast - cp = configparser.ConfigParser() - cp['property'] = config - fd, filename = tempfile.mkstemp( - prefix='config', - suffix='.ini', - dir=tmpdir, - text=True, - ) - with open(fd, 'w') as f: - cp.write(f) - return filename - -class DsEngine(GstEngine): - """ - DeepStream implemetation of GstEngine. - """ - - _tmp = None # type: tempfile.TemporaryDirectory - _previous_scores = None - - def _quit(self): - # cleanup the temporary directory we created on __init__ - self._tmp.cleanup() - # this can self terminate so it should be called last: - super()._quit() - - @property - def tmp(self): - """ - Path to the /tmp/ds_engine... folder used by this engine. - - This path is normally deleted on self._quit() - """ - return self._tmp.name - - _previous_broker_results = None - def on_buffer(self, pad: Gst.Pad, info: Gst.PadProbeInfo, _: None, ) -> Gst.PadProbeReturn: - """ - Get serialized Batch level protobuf string from self._broker - and put it in the result queue. - - connected to the tiler element's sink pad - """ - # get result, and if same as the last, skip it - # yeah, using a GObject property for this is kind - # of odd, but it works. In the future I may make a broker - # and put it in the gi.repository so it'll be easier - # to do this: - proto_str = self._broker.get_property("results") - if proto_str == self._previous_broker_results: - return Gst.PadProbeReturn.OK - self._last_on_buffer_result = proto_str - # we try to update the results queue, but it might be full if - # the results queue is full becauase the ui process is too slow - # (I haven't had this happen, but it covers this) - if not self._update_result_queue(proto_str): - # note: this can hurt performance depending on your logging - # backend (anything that blocks, which is a lot.), but really, - # if we reach this point, something is already hurting, - # and it's probably better to save the data. - self.logger.warning({'dropped_batch_proto': proto_str}) - # NOTE(mdegans): we can drop the whole buffer here if we want to drop - # entire buffers (batches, including images) along with the metadata - # return Gst.PadProbeReturn.DROP - pass - # return pad probe ok, which passes the buffer on - return Gst.PadProbeReturn.OK - - def run(self): - self._tmp = tempfile.TemporaryDirectory(prefix='ds_engine') - super().run() - -if __name__ == "__main__": - import doctest - doctest.testmod(optionflags=doctest.ELLIPSIS) diff --git a/libs/detectors/deepstream/_gst_engine.py b/libs/detectors/deepstream/_gst_engine.py index 0ea4173f..19d7338e 100644 --- a/libs/detectors/deepstream/_gst_engine.py +++ b/libs/detectors/deepstream/_gst_engine.py @@ -89,7 +89,7 @@ def link(a: Gst.Element, b: Gst.Element): if not a.link(b): raise GstLinkError(f'could not link {a.name} to {b.name}') -def link_many(elements: Iterable[Gst.Element]): +def link_many(*elements: Iterable[Gst.Element]): """ Link many Gst.Element. @@ -103,6 +103,7 @@ def link_many(elements: Iterable[Gst.Element]): for current in elements: if not last.link(current): raise GstLinkError(f'could not link {last.name} to {current.name}') + last = current class GstEngine(multiprocessing.Process): @@ -192,7 +193,6 @@ def __init__(self, config:GstConfig, *args, debug=False, blocking=False, **kwarg self._infer_elements = [] # type: List[Gst.Element] self._tracker = None # type: Gst.Element self._distance = None # type: Gst.Element - self._payload = None # type: Gst.Element self._broker = None # type: Gst.Element self._osd_converter = None # type: Gst.Element self._tiler = None # type: Gst.Element @@ -207,49 +207,6 @@ def __init__(self, config:GstConfig, *args, debug=False, blocking=False, **kwarg # so it can be changed after start self.blocking=blocking - @property - def results(self) -> Sequence[str]: - """ - Get results waiting in the queue. - - (may block, depending on self.queue_timeout) - - May return None if no result ready. - - Logs to WARNING level on failure to fetch result. - """ - try: - return self._result_queue.get(block=self.blocking, timeout=self.queue_timeout) - except queue.Empty: - self.logger.warning("failed to get results from queue (queue.Empty)") - return None - except TimeoutError: - self.logger.info("waiting for results...") - return None - - def _update_result_queue(self, results: str): - """ - Called internally by the GStreamer process. - - Update results queue with serialize payload. Should probably be called - by the subclass implemetation of on_buffer(). - - Does not block (because this would block the GLib.MainLoop). - - Can fail if the queue is full in which case the results will - be dropped and logged to the WARNING level. - - Returns: - bool: False on failure, True on success. - """ - if self._result_queue.empty(): - try: - self._result_queue.put_nowait(results) - return True - except queue.Full: - self.logger.warning({'dropped': results}) - return False - def on_bus_message(self, bus: Gst.Bus, message: Gst.Message, *_) -> bool: """ Default bus message callback. @@ -375,6 +332,9 @@ def _create_element(self, e_type:str) -> Optional[Gst.Element]: # set properties on the element if props: + if self._debug: + self.logger.debug( + f'{elem.name}:{props}') for k, v in props.items(): elem.set_property(k, v) @@ -465,7 +425,6 @@ def _create_all(self) -> int: functools.partial(self._create_element, 'tracker'), self._create_infer_elements, functools.partial(self._create_element, 'distance'), - functools.partial(self._create_element, 'payload'), functools.partial(self._create_element, 'broker'), functools.partial(self._create_element, 'osd_converter'), functools.partial(self._create_element, 'tiler'), @@ -527,34 +486,26 @@ def _link_pipeline(self) -> bool: link(self._infer_elements[0], self._tracker) # if there are secondary inference elements if self._infer_elements[1:]: - link_many((self._tracker, *self._infer_elements[1:])) + link_many(self._tracker, *self._infer_elements[1:]) # link the final inference element to distancing engine link(self._infer_elements[-1], self._distance) else: # link tracker directly to the distancing element link(self._tracker, self._distance) - link(self._distance, self._payload) - # TODO(mdegans): rename osd_converter - # (this requires some changes elsewhere) - link(self._payload, self._osd_converter) - link(self._osd_converter, self._tiler) - link(self._tiler, self._osd) - link(self._osd, self._sink) + link_many( + self._distance, + self._broker, + self._osd_converter, + self._tiler, + self._osd, + self._sink, + ) except GstLinkError as err: self.logger.error(f"pipeline link fail because: {err}") return False self.logger.debug('linking pipeline successful') return True - def on_buffer(self, pad: Gst.Pad, info: Gst.PadProbeInfo, _: None, ) -> Gst.PadProbeReturn: - """ - Default source pad probe buffer callback for the sink. - - Simply returns Gst.PadProbeReturn.OK, signaling the buffer - shuould continue down the pipeline. - """ - return Gst.PadProbeReturn.OK - def stop(self): """Stop the GstEngine process.""" self.logger.info('requesting stop') @@ -632,16 +583,6 @@ def run(self): self.logger.error('could not link pipeline') return self._quit() - # register pad probe buffer callback on the tiler - self.logger.debug('registering self.on_buffer() callback on osd sink pad') - tiler_sink_pad = self._tiler.get_static_pad('sink') - if not tiler_sink_pad: - self.logger.error('could not get osd sink pad') - return self._quit() - - self._tiler_probe_id = tiler_sink_pad.add_probe( - Gst.PadProbeType.BUFFER, self.on_buffer, None) - # register callback to check for the stop event when idle. # TODO(mdegans): test to see if a higher priority is needed. self.logger.debug('registering self._on_stop() idle callback with GLib MainLoop') diff --git a/libs/detectors/deepstream/_pyds.py b/libs/detectors/deepstream/_pyds.py deleted file mode 100644 index f5f603aa..00000000 --- a/libs/detectors/deepstream/_pyds.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2020 Michael de Gans -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -Python DeepStream bindings loader. - -This is necessary because Nvidia has no proper python package for the bindings -and a straight-up "import pyds" will not work without sys.path hackery. - -Attributes: - PYDS_PATH (str): - The platform specific path to pyds.so (the bindings). - This path is inserted into sys.path. - PYDS_INSTRUCTIONS (str): - URI for (overly complicated) Installation instructions - for the Python DeepStream bindings. -""" - -import os -import logging -import sys -import platform - -from libs.detectors.deepstream._ds_utils import find_deepstream - -logger = logging.getLogger() - -__all__ = [ - 'PYDS_PATH', - 'PYDS_INSTRUCTIONS', - 'pyds' -] - -# Python DeepStream paths - -DS_INFO = find_deepstream() -if DS_INFO: - DS_ROOT = DS_INFO[1] -else: - raise ImportError( - 'DeepStream not intalled. ' - 'Install with: sudo-apt install deepstream-$VERSION') -PYDS_ROOT = os.path.join(DS_ROOT, 'sources/python/bindings') if DS_ROOT else '/' -PYDS_JETSON_PATH = os.path.join(PYDS_ROOT, 'jetson') -PYDS_x86_64_PATH = os.path.join(PYDS_ROOT, 'x86_64') -# Installing the bindings is actually fairly cumbersome, unfortunately. Please, Nvidia -# put more effort into testing and packaging your products. Speed is not enough. -PYDS_INSTRUCTIONS = 'https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/master/HOWTO.md#running-sample-applications' -if platform.machine() == 'aarch64': - PYDS_PATH = PYDS_JETSON_PATH -elif platform.machine() == 'x86_64': - PYDS_PATH = PYDS_x86_64_PATH -else: - logger.warning( - f"unsupported platform for DeepStream Python bindings") - PYDS_PATH = None - -if PYDS_PATH: - sys.path.insert(0, PYDS_PATH) - try: - import pyds - except ImportError: - logger.warning( - f'pyds could not be imported. ' - f'install instructions: {PYDS_INSTRUCTIONS}') - raise diff --git a/neuralet-distancing.py b/neuralet-distancing.py index d12030a4..32379bd7 100644 --- a/neuralet-distancing.py +++ b/neuralet-distancing.py @@ -1,4 +1,5 @@ #!/usr/bin/python3 +import os import argparse import logging import sys @@ -35,12 +36,10 @@ def main(config, verbose=False): video_path = config.get_section_dict("App").get("VideoPath", None) # create our inference process - try: - # try to launch deepstream but skip if any import errors - # (eg. pyds not found, gi not found) - from libs.detectors.deepstream import DsEngine, DsConfig - process_engine = DsEngine(DsConfig(config)) - except ImportError: + if os.path.isdir('/opt/nvidia/deepstream'): + from libs.detectors.deepstream import GstEngine, DsConfig + process_engine = GstEngine(DsConfig(config), debug=verbose) + else: # DeepStream is not available. Let's try CvEngine process_engine = Process(target=start_cv_engine, args=(config, video_path,)) From 93bb3b4a48746fa55ca9cc8b58b55ee0d53a8ef9 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:14:44 +0000 Subject: [PATCH 10/25] update readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index e726feb4..21ec557e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,14 @@ docker build -f x86-openvino.Dockerfile -t "neuralet/smart-social-distancing:lat docker run -it -p HOST_PORT:8000 -v "$PWD/data":/repo/data neuralet/smart-social-distancing:latest-x86_64_openvino ``` +**Run on x86 using DeepStream** + +``` +cd smart-social-distancing/ +./deepstream.sh build +./deepstream.sh run +``` + ### Configurations You can read and modify the configurations in `config-jetson.ini` file for Jetson Nano and `config-skeleton.ini` file for Coral. From 006b7efa1d5c02cde45f4eba85a46e8509d5f3cd Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:21:04 +0000 Subject: [PATCH 11/25] remove unnecessary argument --- deepstream.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deepstream.sh b/deepstream.sh index 052dd636..641c6f99 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -78,14 +78,13 @@ function build() { function run() { set -x - exec docker run -it --rm \ + exec docker run -it --rm --name smart_distancing \ "${GPU_ARGS[@]}" \ -v "$THIS_DIR/deepstream.ini:/repo/deepstream.ini" \ -v "$THIS_DIR/data:/repo/data" \ --user $UID:$GROUP_ID \ -p "$PORT:8000" \ - "$USER_NAME/smart-distancing:$TAG_SUFFIX1" \ - "$@" + "$USER_NAME/smart-distancing:$TAG_SUFFIX1" } main() { From 5c5c82eb0fd8d28a163ede9573522beca88f5402 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:25:27 +0000 Subject: [PATCH 12/25] single dockerfile --- deepstream-x86_64.Dockerfile => deepstream.Dockerfile | 0 deepstream_docker_build.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename deepstream-x86_64.Dockerfile => deepstream.Dockerfile (100%) diff --git a/deepstream-x86_64.Dockerfile b/deepstream.Dockerfile similarity index 100% rename from deepstream-x86_64.Dockerfile rename to deepstream.Dockerfile diff --git a/deepstream_docker_build.sh b/deepstream_docker_build.sh index 5f64886e..c2ba1e45 100644 --- a/deepstream_docker_build.sh +++ b/deepstream_docker_build.sh @@ -26,7 +26,7 @@ readonly USER_NAME="neuralet" # DeepStream constants: readonly DS_PYBIND_URL="https://developer.nvidia.com/deepstream-getting-started#python_bindings" # Dockerfile names -readonly DOCKERFILE="deepstream-$(arch).Dockerfile" +readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" # change this to use a newer gst-cuda-plugin version From a7c6c47b852981247253b61b6e3be3b8ec7b9132 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 14:27:29 -0700 Subject: [PATCH 13/25] remove unused script, fix correct one --- deepstream.ini | 2 +- deepstream.sh | 2 +- deepstream_docker_build.sh | 70 -------------------------------------- 3 files changed, 2 insertions(+), 72 deletions(-) delete mode 100644 deepstream_docker_build.sh diff --git a/deepstream.ini b/deepstream.ini index f7ba2a71..9de2cb9b 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -3,7 +3,7 @@ Host: 0.0.0.0 Port: 8000 Resolution: 640,480 ; public uri without the trailing slash -PublicUrl: http://localhost:8000 +PublicUrl: http://motherbird:8000 Encoder: nvvideoconvert ! nvv4l2h264enc [Source_0] diff --git a/deepstream.sh b/deepstream.sh index 641c6f99..820785cd 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -30,7 +30,7 @@ readonly USER_NAME="neuralet" # change this to override the arch (should never be necessary) readonly ARCH="$(arch)" # Dockerfile name -readonly DOCKERFILE="deepstream-$ARCH.Dockerfile" +readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" # change this to use a newer gst-cuda-plugin version diff --git a/deepstream_docker_build.sh b/deepstream_docker_build.sh deleted file mode 100644 index c2ba1e45..00000000 --- a/deepstream_docker_build.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020 Michael de Gans -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -set -e - -# change this to your docker hub user if you fork this and want to push it -readonly USER_NAME="neuralet" -# DeepStream constants: -readonly DS_PYBIND_URL="https://developer.nvidia.com/deepstream-getting-started#python_bindings" -# Dockerfile names -readonly DOCKERFILE="deepstream.Dockerfile" -# https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ -readonly THIS_SCRIPT_BASENAME="${0##*/}" -# change this to use a newer gst-cuda-plugin version -readonly CUDA_PLUGIN_VER="0.3.1" - -# get the docker tag suffix from the git branch -TAG_SUFFIX="deepstream-$(git rev-parse --abbrev-ref HEAD)" -if [[ $TAG_SUFFIX == "deepstream-master" ]]; then - # if we're on master, just use "deepstream" - TAG_SUFFIX="deepstream" -fi - -function build() { - exec docker build --pull -f $DOCKERFILE \ - -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ - --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ - . -} - -function run() { - exec docker build --pull -f $DOCKERFILE \ - -t "$USER_NAME/smart-distancing:$TAG_SUFFIX-$1" \ - --build-arg CUDA_PLUGIN_TAG="${CUDA_PLUGIN_VER}-$1" \ - . -} - -main() { - local ARCH="$(arch)" -case "$1" in - build) - build $ARCH - ;; - run) - run $ARCH - ;; - *) - echo "Usage: $ARCH" -esac -} - -main "$1" \ No newline at end of file From 21de1ca02360fa3cf50794273b2befc2f74cf67f Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 14:29:03 -0700 Subject: [PATCH 14/25] undo hostname rename --- deepstream.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.ini b/deepstream.ini index 9de2cb9b..f7ba2a71 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -3,7 +3,7 @@ Host: 0.0.0.0 Port: 8000 Resolution: 640,480 ; public uri without the trailing slash -PublicUrl: http://motherbird:8000 +PublicUrl: http://localhost:8000 Encoder: nvvideoconvert ! nvv4l2h264enc [Source_0] From 38c9387fe65c26c5676229cc57a64677e03dbfa9 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 14:34:02 -0700 Subject: [PATCH 15/25] fix usage --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21ec557e..16dda290 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,8 @@ docker run -it -p HOST_PORT:8000 -v "$PWD/data":/repo/data neuralet/smart-social ``` cd smart-social-distancing/ -./deepstream.sh build -./deepstream.sh run +(sudo) ./deepstream.sh build +(sudo) ./deepstream.sh run ``` ### Configurations From da76665cba60c41a3bfbf807de3ca5a0941dcabf Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 15:16:50 -0700 Subject: [PATCH 16/25] change requirements for tegra --- deepstream.Dockerfile | 9 +++- requirements.in | 6 +-- requirements.txt | 110 ++++++++---------------------------------- 3 files changed, 27 insertions(+), 98 deletions(-) diff --git a/deepstream.Dockerfile b/deepstream.Dockerfile index fc9e7fec..f8f1f200 100644 --- a/deepstream.Dockerfile +++ b/deepstream.Dockerfile @@ -33,12 +33,17 @@ COPY requirements.txt /tmp/ # install pip, install requirements, remove pip and deps RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-aiofiles \ + python3-dev \ python3-gi \ python3-gst-1.0 \ + python3-numpy \ + python3-opencv \ + python3-pil \ python3-pip \ + python3-protobuf \ + python3-scipy \ python3-setuptools \ - python3-opencv \ - python3-dev \ graphviz \ && pip3 install --require-hashes -r /tmp/requirements.txt \ && apt-get purge -y --autoremove \ diff --git a/requirements.in b/requirements.in index 2455770d..ca36102c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,8 +1,4 @@ -aiofiles fastapi -numpy -Pillow -protobuf -scipy +protobuf==3.12.2 setuptools uvicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 44385535..6faa6b26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,6 @@ # # pip-compile --allow-unsafe --generate-hashes # -aiofiles==0.5.0 \ - --hash=sha256:377fdf7815cc611870c59cbd07b68b180841d2a2b79812d8c218be02448c2acb \ - --hash=sha256:98e6bcfd1b50f97db4980e182ddd509b7cc35909e903a8fe50d8849e02d815af \ - # via -r requirements.in click==7.1.2 \ --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \ @@ -38,70 +34,25 @@ httptools==0.1.1 \ --hash=sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4 \ --hash=sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437 \ # via uvicorn -numpy==1.18.3 \ - --hash=sha256:0aa2b318cf81eb1693fcfcbb8007e95e231d7e1aa24288137f3b19905736c3ee \ - --hash=sha256:163c78c04f47f26ca1b21068cea25ed7c5ecafe5f5ab2ea4895656a750582b56 \ - --hash=sha256:1e37626bcb8895c4b3873fcfd54e9bfc5ffec8d0f525651d6985fcc5c6b6003c \ - --hash=sha256:264fd15590b3f02a1fbc095e7e1f37cdac698ff3829e12ffdcffdce3772f9d44 \ - --hash=sha256:3d9e1554cd9b5999070c467b18e5ae3ebd7369f02706a8850816f576a954295f \ - --hash=sha256:40c24960cd5cec55222963f255858a1c47c6fa50a65a5b03fd7de75e3700eaaa \ - --hash=sha256:46f404314dbec78cb342904f9596f25f9b16e7cf304030f1339e553c8e77f51c \ - --hash=sha256:4847f0c993298b82fad809ea2916d857d0073dc17b0510fbbced663b3265929d \ - --hash=sha256:48e15612a8357393d176638c8f68a19273676877caea983f8baf188bad430379 \ - --hash=sha256:6725d2797c65598778409aba8cd67077bb089d5b7d3d87c2719b206dc84ec05e \ - --hash=sha256:99f0ba97e369f02a21bb95faa3a0de55991fd5f0ece2e30a9e2eaebeac238921 \ - --hash=sha256:a41f303b3f9157a31ce7203e3ca757a0c40c96669e72d9b6ee1bce8507638970 \ - --hash=sha256:a4305564e93f5c4584f6758149fd446df39fd1e0a8c89ca0deb3cce56106a027 \ - --hash=sha256:a551d8cc267c634774830086da42e4ba157fa41dd3b93982bc9501b284b0c689 \ - --hash=sha256:a6bc9432c2640b008d5f29bad737714eb3e14bb8854878eacf3d7955c4e91c36 \ - --hash=sha256:c60175d011a2e551a2f74c84e21e7c982489b96b6a5e4b030ecdeacf2914da68 \ - --hash=sha256:e46e2384209c91996d5ec16744234d1c906ab79a701ce1a26155c9ec890b8dc8 \ - --hash=sha256:e607b8cdc2ae5d5a63cd1bec30a15b5ed583ac6a39f04b7ba0f03fcfbf29c05b \ - --hash=sha256:e94a39d5c40fffe7696009dbd11bc14a349b377e03a384ed011e03d698787dd3 \ - --hash=sha256:eb2286249ebfe8fcb5b425e5ec77e4736d53ee56d3ad296f8947f67150f495e3 \ - --hash=sha256:fdee7540d12519865b423af411bd60ddb513d2eb2cd921149b732854995bbf8b \ - # via -r requirements.in, scipy -pillow==7.1.2 \ - --hash=sha256:04766c4930c174b46fd72d450674612ab44cca977ebbcc2dde722c6933290107 \ - --hash=sha256:0e2a3bceb0fd4e0cb17192ae506d5f082b309ffe5fc370a5667959c9b2f85fa3 \ - --hash=sha256:0f01e63c34f0e1e2580cc0b24e86a5ccbbfa8830909a52ee17624c4193224cd9 \ - --hash=sha256:12e4bad6bddd8546a2f9771485c7e3d2b546b458ae8ff79621214119ac244523 \ - --hash=sha256:1f694e28c169655c50bb89a3fa07f3b854d71eb47f50783621de813979ba87f3 \ - --hash=sha256:3d25dd8d688f7318dca6d8cd4f962a360ee40346c15893ae3b95c061cdbc4079 \ - --hash=sha256:4b02b9c27fad2054932e89f39703646d0c543f21d3cc5b8e05434215121c28cd \ - --hash=sha256:9744350687459234867cbebfe9df8f35ef9e1538f3e729adbd8fde0761adb705 \ - --hash=sha256:a0b49960110bc6ff5fead46013bcb8825d101026d466f3a4de3476defe0fb0dd \ - --hash=sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3 \ - --hash=sha256:b37bb3bd35edf53125b0ff257822afa6962649995cbdfde2791ddb62b239f891 \ - --hash=sha256:b532bcc2f008e96fd9241177ec580829dee817b090532f43e54074ecffdcd97f \ - --hash=sha256:b67a6c47ed963c709ed24566daa3f95a18f07d3831334da570c71da53d97d088 \ - --hash=sha256:b943e71c2065ade6fef223358e56c167fc6ce31c50bc7a02dd5c17ee4338e8ac \ - --hash=sha256:ccc9ad2460eb5bee5642eaf75a0438d7f8887d484490d5117b98edd7f33118b7 \ - --hash=sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d \ - --hash=sha256:eaa83729eab9c60884f362ada982d3a06beaa6cc8b084cf9f76cae7739481dfa \ - --hash=sha256:ee94fce8d003ac9fd206496f2707efe9eadcb278d94c271f129ab36aa7181344 \ - --hash=sha256:f455efb7a98557412dc6f8e463c1faf1f1911ec2432059fa3e582b6000fc90e2 \ - --hash=sha256:f46e0e024346e1474083c729d50de909974237c72daca05393ee32389dabe457 \ - --hash=sha256:f54be399340aa602066adb63a86a6a5d4f395adfdd9da2b9a0162ea808c7b276 \ - --hash=sha256:f784aad988f12c80aacfa5b381ec21fd3f38f851720f652b9f33facc5101cf4d \ - # via -r requirements.in -protobuf==3.12.1 \ - --hash=sha256:04d0b2bd99050d09393875a5a25fd12337b17f3ac2e29c0c1b8e65b277cbfe72 \ - --hash=sha256:05288e44638e91498f13127a3699a6528dec6f9d3084d60959d721bfb9ea5b98 \ - --hash=sha256:175d85370947f89e33b3da93f4ccdda3f326bebe3e599df5915ceb7f804cd9df \ - --hash=sha256:440a8c77531b3652f24999b249256ed01fd44c498ab0973843066681bd276685 \ - --hash=sha256:49fb6fab19cd3f30fa0e976eeedcbf2558e9061e5fa65b4fe51ded1f4002e04d \ - --hash=sha256:4c7cae1f56056a4a2a2e3b00b26ab8550eae738bd9548f4ea0c2fcb88ed76ae5 \ - --hash=sha256:519abfacbb421c3591d26e8daf7a4957763428db7267f7207e3693e29f6978db \ - --hash=sha256:60f32af25620abc4d7928d8197f9f25d49d558c5959aa1e08c686f974ac0b71a \ - --hash=sha256:613ac49f6db266fba243daf60fb32af107cfe3678e5c003bb40a381b6786389d \ - --hash=sha256:954bb14816edd24e746ba1a6b2d48c43576393bbde2fb8e1e3bd6d4504c7feac \ - --hash=sha256:9b1462c033a2cee7f4e8eb396905c69de2c532c3b835ff8f71f8e5fb77c38023 \ - --hash=sha256:c0767f4d93ce4288475afe0571663c78870924f1f8881efd5406c10f070c75e4 \ - --hash=sha256:c45f5980ce32879391144b5766120fd7b8803129f127ce36bd060dd38824801f \ - --hash=sha256:eeb7502f59e889a88bcb59f299493e215d1864f3d75335ea04a413004eb4fe24 \ - --hash=sha256:fdb1742f883ee4662e39fcc5916f2725fec36a5191a52123fec60f8c53b70495 \ - --hash=sha256:fe554066c4962c2db0a1d4752655223eb948d2bfa0fb1c4a7f2c00ec07324f1c \ +protobuf==3.12.2 \ + --hash=sha256:304e08440c4a41a0f3592d2a38934aad6919d692bb0edfb355548786728f9a5e \ + --hash=sha256:49ef8ab4c27812a89a76fa894fe7a08f42f2147078392c0dee51d4a444ef6df5 \ + --hash=sha256:50b5fee674878b14baea73b4568dc478c46a31dd50157a5b5d2f71138243b1a9 \ + --hash=sha256:5524c7020eb1fb7319472cb75c4c3206ef18b34d6034d2ee420a60f99cddeb07 \ + --hash=sha256:612bc97e42b22af10ba25e4140963fbaa4c5181487d163f4eb55b0b15b3dfcd2 \ + --hash=sha256:6f349adabf1c004aba53f7b4633459f8ca8a09654bf7e69b509c95a454755776 \ + --hash=sha256:85b94d2653b0fdf6d879e39d51018bf5ccd86c81c04e18a98e9888694b98226f \ + --hash=sha256:87535dc2d2ef007b9d44e309d2b8ea27a03d2fa09556a72364d706fcb7090828 \ + --hash=sha256:a7ab28a8f1f043c58d157bceb64f80e4d2f7f1b934bc7ff5e7f7a55a337ea8b0 \ + --hash=sha256:a96f8fc625e9ff568838e556f6f6ae8eca8b4837cdfb3f90efcb7c00e342a2eb \ + --hash=sha256:b5a114ea9b7fc90c2cc4867a866512672a47f66b154c6d7ee7e48ddb68b68122 \ + --hash=sha256:be04fe14ceed7f8641e30f36077c1a654ff6f17d0c7a5283b699d057d150d82a \ + --hash=sha256:bff02030bab8b969f4de597543e55bd05e968567acb25c0a87495a31eb09e925 \ + --hash=sha256:c9ca9f76805e5a637605f171f6c4772fc4a81eced4e2f708f79c75166a2c99ea \ + --hash=sha256:e1464a4a2cf12f58f662c8e6421772c07947266293fb701cb39cd9c1e183f63c \ + --hash=sha256:e72736dd822748b0721f41f9aaaf6a5b6d5cfc78f6c8690263aef8bba4457f0e \ + --hash=sha256:eafe9fa19fcefef424ee089fb01ac7177ff3691af7cc2ae8791ae523eb6ca907 \ + --hash=sha256:f4b73736108a416c76c17a8a09bc73af3d91edaa26c682aaa460ef91a47168d3 \ # via -r requirements.in pydantic==1.5.1 \ --hash=sha256:0a1cdf24e567d42dc762d3fed399bd211a13db2e8462af9dfa93b34c41648efb \ @@ -122,29 +73,6 @@ pydantic==1.5.1 \ --hash=sha256:e2c753d355126ddd1eefeb167fa61c7037ecd30b98e7ebecdc0d1da463b4ea09 \ --hash=sha256:f0018613c7a0d19df3240c2a913849786f21b6539b9f23d85ce4067489dfacfa \ # via fastapi -scipy==1.4.1 \ - --hash=sha256:00af72998a46c25bdb5824d2b729e7dabec0c765f9deb0b504f928591f5ff9d4 \ - --hash=sha256:0902a620a381f101e184a958459b36d3ee50f5effd186db76e131cbefcbb96f7 \ - --hash=sha256:1e3190466d669d658233e8a583b854f6386dd62d655539b77b3fa25bfb2abb70 \ - --hash=sha256:2cce3f9847a1a51019e8c5b47620da93950e58ebc611f13e0d11f4980ca5fecb \ - --hash=sha256:3092857f36b690a321a662fe5496cb816a7f4eecd875e1d36793d92d3f884073 \ - --hash=sha256:386086e2972ed2db17cebf88610aab7d7f6e2c0ca30042dc9a89cf18dcc363fa \ - --hash=sha256:71eb180f22c49066f25d6df16f8709f215723317cc951d99e54dc88020ea57be \ - --hash=sha256:770254a280d741dd3436919d47e35712fb081a6ff8bafc0f319382b954b77802 \ - --hash=sha256:787cc50cab3020a865640aba3485e9fbd161d4d3b0d03a967df1a2881320512d \ - --hash=sha256:8a07760d5c7f3a92e440ad3aedcc98891e915ce857664282ae3c0220f3301eb6 \ - --hash=sha256:8d3bc3993b8e4be7eade6dcc6fd59a412d96d3a33fa42b0fa45dc9e24495ede9 \ - --hash=sha256:9508a7c628a165c2c835f2497837bf6ac80eb25291055f56c129df3c943cbaf8 \ - --hash=sha256:a144811318853a23d32a07bc7fd5561ff0cac5da643d96ed94a4ffe967d89672 \ - --hash=sha256:a1aae70d52d0b074d8121333bc807a485f9f1e6a69742010b33780df2e60cfe0 \ - --hash=sha256:a2d6df9eb074af7f08866598e4ef068a2b310d98f87dc23bd1b90ec7bdcec802 \ - --hash=sha256:bb517872058a1f087c4528e7429b4a44533a902644987e7b2fe35ecc223bc408 \ - --hash=sha256:c5cac0c0387272ee0e789e94a570ac51deb01c796b37fb2aad1fb13f85e2f97d \ - --hash=sha256:cc971a82ea1170e677443108703a2ec9ff0f70752258d0e9f5433d00dda01f59 \ - --hash=sha256:dba8306f6da99e37ea08c08fef6e274b5bf8567bb094d1dbe86a20e532aca088 \ - --hash=sha256:dc60bb302f48acf6da8ca4444cfa17d52c63c5415302a9ee77b3b21618090521 \ - --hash=sha256:dee1bbf3a6c8f73b6b218cb28eed8dd13347ea2f87d572ce19b289d6fd3fbc59 \ - # via -r requirements.in six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ From 3ffc2fb4df72e2cd7ddba04ba6b3a973cfda9f03 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 15:39:30 -0700 Subject: [PATCH 17/25] fix missing video * I'm getting a 404 trying to download it --- deepstream.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.ini b/deepstream.ini index f7ba2a71..1959dafd 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -10,7 +10,7 @@ Encoder: nvvideoconvert ! nvv4l2h264enc ; VideoPath may be a uri supported by uridecodebin (rtsp, http, etc.) ; or a local file. ; TODO(mdegans): camera sources. -VideoPath: /repo/data/TownCentreXVID.avi +VideoPath: /opt/nvidia/deepstream/deepstream-5.0/samples/streams/sample_720p.h264 [Source_1] VideoPath: /opt/nvidia/deepstream/deepstream-5.0/samples/streams/sample_720p.h264 From 1424782daf9d489fcc1a52aa60bd07a15ae7cacb Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Thu, 18 Jun 2020 15:39:51 -0700 Subject: [PATCH 18/25] fix wrong user when running with sudo --- deepstream.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deepstream.sh b/deepstream.sh index 820785cd..93f97fe1 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -52,6 +52,13 @@ else "all" ) fi +# the user id to use +if [[ -z "$SUDO_USER" ]]; then + readonly USER_ID=$UID +else + echo "sudo user: $SUDO_USER" + readonly USER_ID="$(id -u $SUDO_USER)" +fi # this helps tag the image GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -82,9 +89,10 @@ function run() { "${GPU_ARGS[@]}" \ -v "$THIS_DIR/deepstream.ini:/repo/deepstream.ini" \ -v "$THIS_DIR/data:/repo/data" \ - --user $UID:$GROUP_ID \ + --user $USER_ID:$GROUP_ID \ -p "$PORT:8000" \ - "$USER_NAME/smart-distancing:$TAG_SUFFIX1" + "$USER_NAME/smart-distancing:$TAG_SUFFIX1" \ + --verbose } main() { From b7840b1bbe40bb7664f7dbbeaf6e481c668ff1a6 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 19 Jun 2020 15:11:47 -0700 Subject: [PATCH 19/25] use correct frontend --- deepstream.Dockerfile | 12 +++++++----- deepstream.sh | 9 ++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/deepstream.Dockerfile b/deepstream.Dockerfile index f8f1f200..124bef87 100644 --- a/deepstream.Dockerfile +++ b/deepstream.Dockerfile @@ -18,7 +18,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -ARG CUDA_PLUGIN_TAG="build me with deepstream_docker_build.sh" +ARG FRONTEND_BASE="build me with deesptream.sh" +ARG CUDA_PLUGIN_TAG="build me with deepstream.sh" +FROM ${FRONTEND_BASE} as frontend FROM registry.hub.docker.com/mdegans/gstcudaplugin:${CUDA_PLUGIN_TAG} # this can't be downloaded directly because a license needs to be accepted, @@ -62,7 +64,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /repo -COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /srv/frontend +# copy frontend +COPY --from=frontend /frontend/build /srv/frontend + +# copy code COPY neuralet-distancing.py README.md ${CONFIG_FILE} ./ COPY libs ./libs/ COPY ui ./ui/ @@ -70,9 +75,6 @@ COPY tools ./tools/ COPY logs ./logs/ COPY data ./data/ -# copy frontend -COPY --from=neuralet/smart-social-distancing:latest-frontend /frontend/build /srv/frontend - # entrypoint with deepstream. EXPOSE 8000 ENTRYPOINT [ "/usr/bin/python3", "neuralet-distancing.py", "--config", "deepstream.ini" ] diff --git a/deepstream.sh b/deepstream.sh index 93f97fe1..95c67bd9 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -29,6 +29,8 @@ readonly VERSION="0.1.0" readonly USER_NAME="neuralet" # change this to override the arch (should never be necessary) readonly ARCH="$(arch)" +# frontend dockerfile name +readonly FRONTEND_DOCKERFILE="frontend.Dockerfile" # Dockerfile name readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ @@ -75,11 +77,16 @@ readonly TAG_SUFFIX1="deepstream-$VERSION-$ARCH" readonly TAG_SUFFIX2="deepstream-$GIT_BRANCH-$ARCH" function build() { + readonly local FRONTEND_TAG="$GIT_BRANCH-frontend" set -x - exec docker build --pull -f $DOCKERFILE \ + docker build -f $FRONTEND_DOCKERFILE \ + -t "$USER_NAME/smart-social-distancing:$FRONTEND_TAG" \ + . + docker build -f $DOCKERFILE \ -t "$USER_NAME/smart-distancing:$TAG_SUFFIX1" \ -t "$USER_NAME/smart-distancing:$TAG_SUFFIX2" \ --build-arg CUDA_PLUGIN_TAG="$CUDA_PLUGIN_VER-$ARCH" \ + --build-arg FRONTEND_BASE="$USER_NAME/smart-social-distancing:$FRONTEND_TAG" \ . } From c92b0656c6e44d6d79d3a135d495769a9ddeec56 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 19 Jun 2020 16:02:20 -0700 Subject: [PATCH 20/25] remove OpenCV * good riddance --- deepstream.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/deepstream.Dockerfile b/deepstream.Dockerfile index 124bef87..7a97d8b3 100644 --- a/deepstream.Dockerfile +++ b/deepstream.Dockerfile @@ -40,7 +40,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-gi \ python3-gst-1.0 \ python3-numpy \ - python3-opencv \ python3-pil \ python3-pip \ python3-protobuf \ From 799c5e87e8a543928902e4e7df673314ed34d45f Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 19 Jun 2020 16:02:57 -0700 Subject: [PATCH 21/25] bump plugin base layer * to patch with fixed precision float in .csv --- deepstream.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.sh b/deepstream.sh index 95c67bd9..ba48ff2a 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -36,7 +36,7 @@ readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" # change this to use a newer gst-cuda-plugin version -readonly CUDA_PLUGIN_VER="0.3.1" +readonly CUDA_PLUGIN_VER="0.3.2" # https://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself readonly THIS_DIR="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" # the primary group to use From e609378559b92845ee4d823870b4e6e1fe1d89c1 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 19 Jun 2020 17:19:53 -0700 Subject: [PATCH 22/25] fix metadata by using a newer gst-cuda-plugin --- deepstream.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.sh b/deepstream.sh index ba48ff2a..948ad9ad 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -36,7 +36,7 @@ readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" # change this to use a newer gst-cuda-plugin version -readonly CUDA_PLUGIN_VER="0.3.2" +readonly CUDA_PLUGIN_VER="0.3.3" # https://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself readonly THIS_DIR="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" # the primary group to use From d91a5869ed51e55ff1f7b52920d7467a070f63b2 Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Fri, 19 Jun 2020 17:22:51 -0700 Subject: [PATCH 23/25] hostname note --- README.md | 5 +++++ deepstream.ini | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16dda290..f7862a17 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ cd smart-social-distancing/ (sudo) ./deepstream.sh run ``` +Make sure to set your hostname to your publicly accessable hostname in deepstream.ini +``` +PublicUrl: http://your_hostname_here:8000 +``` + ### Configurations You can read and modify the configurations in `config-jetson.ini` file for Jetson Nano and `config-skeleton.ini` file for Coral. diff --git a/deepstream.ini b/deepstream.ini index 1959dafd..0fee256b 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -3,7 +3,7 @@ Host: 0.0.0.0 Port: 8000 Resolution: 640,480 ; public uri without the trailing slash -PublicUrl: http://localhost:8000 +PublicUrl: http://motherbird:8000 Encoder: nvvideoconvert ! nvv4l2h264enc [Source_0] From 2d60cadbd9d2b9298cb1df2c1216fcddaec8963a Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Mon, 22 Jun 2020 14:23:24 -0700 Subject: [PATCH 24/25] fix hostname --- deepstream.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.ini b/deepstream.ini index 0fee256b..1959dafd 100644 --- a/deepstream.ini +++ b/deepstream.ini @@ -3,7 +3,7 @@ Host: 0.0.0.0 Port: 8000 Resolution: 640,480 ; public uri without the trailing slash -PublicUrl: http://motherbird:8000 +PublicUrl: http://localhost:8000 Encoder: nvvideoconvert ! nvv4l2h264enc [Source_0] From 9773913dff3d3982fbf67452d358dcc080c7a4cd Mon Sep 17 00:00:00 2001 From: Michael de Gans <47511965+mdegans@users.noreply.github.com> Date: Mon, 22 Jun 2020 14:26:11 -0700 Subject: [PATCH 25/25] bump cuda plugin version --- deepstream.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepstream.sh b/deepstream.sh index 948ad9ad..fc9bc86a 100755 --- a/deepstream.sh +++ b/deepstream.sh @@ -36,7 +36,7 @@ readonly DOCKERFILE="deepstream.Dockerfile" # https://www.cyberciti.biz/faq/bash-get-basename-of-filename-or-directory-name/ readonly THIS_SCRIPT_BASENAME="${0##*/}" # change this to use a newer gst-cuda-plugin version -readonly CUDA_PLUGIN_VER="0.3.3" +readonly CUDA_PLUGIN_VER="0.3.4" # https://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself readonly THIS_DIR="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" # the primary group to use