Merge branch 'master' into fix_launch_and_results

Author: ismukhin

.github/workflows/codestyle.yml

@@ -6,7 +6,7 @@ jobs:
   codestyle:
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Code checkout

.github/workflows/smoke_test.yml

@@ -12,7 +12,7 @@ jobs:
   smoke_test_benchmark:
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Code checkout
@@ -33,19 +33,20 @@ jobs:
         run: cd tests/smoke_test/benchmark_smoke && python -m pytest test_benchmark_smoke.py -m="not caffe"
 
       - name: Upload benchmark artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: benchmark_report
           path: tests/smoke_test/benchmark_smoke/report.html
           retention-days: 1
 
       - name: Run smoke tests for accuracy checker
+        if: false
         run: cd tests/smoke_test/ac_smoke && python -m pytest test_ac_smoke.py
 
       - name: Upload accuracy artifacts
-        uses: actions/upload-artifact@v3
-        if: always()
+        uses: actions/upload-artifact@v4
+        if: false
         with:
           name: accuracy_report
           path: tests/smoke_test/ac_smoke/report.html
@@ -54,7 +55,7 @@ jobs:
   smoke_test_benchmark_caffe:
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Code checkout
@@ -83,17 +84,17 @@ jobs:
           cd tests/smoke_test/benchmark_smoke && python -m pytest test_benchmark_smoke.py -m="caffe"
 
       - name: Upload benchmark caffe artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: benchmark_caffe_results
+          name: benchmark_report_caffe
           path: tests/smoke_test/benchmark_smoke/report.html
           retention-days: 1
 
   smoke_test_quantization:
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Code checkout
@@ -114,9 +115,9 @@ jobs:
         run: cd tests/smoke_test/quantization_smoke && python -m pytest test_quantization_smoke.py
 
       - name: Upload quantization artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: benchmark_report
+          name: quantization_report
           path: tests/smoke_test/quantization_smoke/report.html
           retention-days: 1

.github/workflows/static.yml

@@ -31,12 +31,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup Pages
-        uses: actions/configure-pages@v2
+        uses: actions/configure-pages@v4
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           # Upload entire repository
           path: '.'
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/unit_test.yml

@@ -12,7 +12,7 @@ jobs:
   unit_test:
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Code checkout

README.md

@@ -32,6 +32,7 @@ DLI supports inference using the following frameworks:
 - [RKNN][rknn] (C++ API).
 - [ncnn][ncnn] (Python API).
 - [PaddlePaddle][PaddlePaddle] (Python API).
+- [ExecuTorch][executorch] (C++ API)
 
 More information about DLI is available on the web-site
 ([here][dli-ru-web-page] (in Russian)
@@ -47,8 +48,8 @@ Please consider citing the following papers.
 
 1. Kustikova V., Vasilyev E., Khvatov A., Kumbrasiev P., Rybkin R.,
    Kogteva N. DLI: Deep Learning Inference Benchmark //
-   Communications in Computer and Information Science.
-   V.1129. 2019. P. 542-553.
+   Communications in Computer and Information
+   Science. - 2019. - V. 1129. - P. 542-553.
 
 1. Sidorova A.K.,  Alibekov M.R., Makarov A.A., Vasiliev E.P., 
    Kustikova V.D. Automation of collecting performance indicators 
@@ -73,10 +74,11 @@ Please consider citing the following papers.
    and Programming. - 2024. - Vol. 25(2). - P. 127-141. -
    [https://num-meth.ru/index.php/journal/article/view/1332/1264][nummeth2023].
    (In Russian)
+
 1. Mukhin I., Rodimkov Y., Vasiliev E., Volokitin V., Sidorova A.,
    Kozinov E., Meyerov I., Kustikova V. Benchmarking Deep Learning
    Inference on RISC-V CPUs // Springer Lecture Notes in Computer
-   Science. – 2024. – Accepted.
+   Science. – 2025. – Vol. 15406. - P. 331-346.
 
 ## Repo structure
 
@@ -97,12 +99,12 @@ Please consider citing the following papers.
   - `OpenCV` is a directory of Dockerfiles for OpenCV.
   - `OpenVINO_DLDT` is a directory of Dockerfiles for Intel®
     Distribution of OpenVINO™ Toolkit.
+  - `PaddlePaddle` is a directory of Dockerfiles for PaddlePaddle.
   - `PyTorch` is a directory of Dockerfiles for PyTorch.
-  - `TVM` is a directory of Dockerfiles for Apache TVM.
   - `TensorFlow` is a directory of Dockerfiles for Intel® Optimizations
     for TensorFlow.
   - `TensorFlowLite` is a directory of Dockerfiles for TensorFlow Lite.
-  - `PaddlePaddle` is a directory of Dockerfiles for PaddlePaddle.
+  - `TVM` is a directory of Dockerfiles for Apache TVM.
 
 - `docs` directory contains auxiliary documentation. Please, find
   complete documentation at the [Wiki page][dli-wiki].
@@ -138,6 +140,9 @@ Please consider citing the following papers.
       is a table that confirms correctness of inference implementation
       based on Intel® Distribution of OpenVINO™ toolkit for models trained
       by Intel engineers and available in [Open Model Zoo][open-model-zoo].
+    - [`validation_results_paddlepaddle.md`](results/validation/validation_results_paddlepaddle.md)
+      is a table that confirms correctness of inference implementation
+      based on PaddlePaddle.
     - [`validation_results_pytorch.md`](results/validation/validation_results_pytorch.md)
       is a table that confirms correctness of inference implementation
       based on PyTorch for [TorchVision][torchvision].
@@ -161,9 +166,11 @@ Please consider citing the following papers.
   - [`onnxruntime_models_checklist.md`](results/onnxruntime_models_checklist.md) contains a list
     of deep models inferred by ONNX Runtime checked in the DLI benchmark.
   - [`opencv_models_checklist.md`](results/opencv_models_checklist.md) contains a list
-    of deep models inferred by OpenCV DNN.
+    of deep models inferred by OpenCV DNN checked in the DLI benchmark.
   - [`openvino_models_checklist.md`](results/openvino_models_checklist.md) contains a list
     of deep models inferred by the OpenVINO toolkit checked in the DLI benchmark.
+  - [`paddlepaddle_models_checklist.md`](results/paddlepaddle_models_checklist.md)
+    containsa list deep models inferred by PaddlePaddle checked in the DLI benchmark.
   - [`pytorch_models_checklist.md`](results/pytorch_models_checklist.md) contains a list
     of deep models inferred by PyTorch checked in the DLI benchmark.
   - [`tensorflow_models_checklist.md`](results/tensorflow_models_checklist.md) contains a list
@@ -179,14 +186,17 @@ Please consider citing the following papers.
     using Accuracy Checker of Intel® Distribution of OpenVINO™ toolkit.
   - `benchmark` is a set of scripts to estimate inference
     performance of different models at the single local computer.
-  - `build_scripts` is a directory to build inference frameworks for different platforms.
+  - `build_scripts` is a directory to build inference frameworks for different
+    platforms.
   - `config_maker`contains GUI-application to make configuration files
-    of the benchmark components.
+    of the benchmark components. Application supports outdated version
+    of configuration files. It is required to update (one of the future tasks).
   - `configs` contains template configuration files.
   - `cpp_dl_benchmark` contains C++ tools that allow to measure
     deep learning models inference performance with
     [ONNX Runtime][onnx-runtime-github], [OpenCV DNN][opencv-dnn],
-    [PyTorch][pytorch] and [TensorFlow Lite][tensorflow-lite] in C++ API implementation.
+    [PyTorch][pytorch], [ExecuTorch][executorch] and 
+    [TensorFlow Lite][tensorflow-lite] in C++ API implementation.
     This implementation inspired by [OpenVINO Benchmark C++ tool][benchmark-app]
     as a reference and stick to its measurement methodology,
     thus provide consistent performance results.
@@ -199,15 +209,15 @@ Please consider citing the following papers.
   - `model_converters` contains converters of deep models.
   - `node_info` contains a set of functions to get information about
     computational node.
-  - `quantization` contains scripts to quantize model to INT8-precision
-    using Post-Training Optimization Tool (POT)
-    of Intel® Distribution of OpenVINO™ toolkit.
+  - `quantization` contains scripts to quantize model
+    using Post-Training Optimization Tool (POT) of Intel® Distribution
+    of OpenVINO™ toolkit, TensorFlow Lite and TVM internal tools.
   - `remote_control` contains scripts to execute benchmark
     remotely.
   - `tvm_autotuning` contains scripts to optimize Apache TVM models.
   - `utils` is a package of auxiliary utilities.
 
-- `test` contains smoke tests.
+- `tests` contains smoke tests.
 
 - `requirements.txt` is a list of special requirements for the DLI
   benchmark without inference frameworks.
@@ -271,6 +281,7 @@ Report questions, issues and suggestions, using:
 [rknn]: https://github.com/rockchip-linux/rknn-toolkit2
 [ncnn]: https://github.com/Tencent/ncnn
 [PaddlePaddle]: https://www.paddlepaddle.org.cn/en
+[executorch]: https://pytorch.org/executorch-overview
 [benchmark-app]: https://github.com/openvinotoolkit/openvino/tree/master/samples/cpp/benchmark_app
 [dli-ru-web-page]: http://hpc-education.unn.ru/dli-ru
 [dli-web-page]: http://hpc-education.unn.ru/dli

demo/benchmark_configs/TVM.xml

@@ -30,7 +30,7 @@
             <ChannelSwap></ChannelSwap>
             <Target>llvm</Target>
             <Layout>NCHW</Layout>
-            <VirtualMachine>False</VirtualMachine>
+            <HighLevelAPI>Relay</HighLevelAPI>
             <OptimizationLevel>3</OptimizationLevel>
         </FrameworkDependent>
     </Test>

docker/TVM/Dockerfile

@@ -5,37 +5,60 @@ WORKDIR /root/
 # Installing miniconda
 RUN wget -q --no-check-certificate -c https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash Miniconda3-latest-Linux-x86_64.sh -b && \
-    ./miniconda3/bin/conda create -n tvm-env -y python=3.7.16 && \
+    ./miniconda3/bin/conda create -n tvm-env -y python=3.8.20 && \
     rm -rf /root/miniconda3/pkgs/* && \
     rm ~/Miniconda3-latest-Linux-x86_64.sh -f
-    
+
 ENV PATH /root/miniconda3/envs/tvm-env/bin:/root/miniconda3/bin:$PATH
 RUN echo "source activate tvm-env" > ~/.bashrc
 RUN export LD_LIBRARY_PATH=/root/miniconda3/envs/tvm-env/lib:${LD_LIBRARY_PATH}
 RUN conda config --add channels intel
 
 # Installing dependencies
-RUN python3 -m pip install pycocotools docker PyYAML
+RUN python3 -m pip install pycocotools docker PyYAML gluoncv[full] opencv-python Cython psutil
 RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6
-RUN python3 -m pip install gluoncv[full]
-RUN python3 -m pip install opencv-python
-   
+RUN apt-get update && apt-get install -y -qq --no-install-recommends cmake software-properties-common
+RUN apt-add-repository "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main" && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+RUN apt-get update && \
+    apt-get install -y -qq --no-install-recommends llvm-19 llvm-19-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 # Installing Apache-TVM 
-RUN python3 -m pip install apache-tvm==0.14.dev264
+ARG TVM_VERSION=v0.19.0
+
+RUN git clone --recursive https://github.com/apache/tvm tvm --branch ${TVM_VERSION} --single-branch
+ENV TVM_BUILD_DIR=/tmp/build-tvm
+RUN mkdir $TVM_BUILD_DIR && cd $TVM_BUILD_DIR
+RUN cp /root/tvm/cmake/config.cmake .
+RUN echo "set(CMAKE_BUILD_TYPE RelWithDebInfo)" >> config.cmake
+RUN echo "set(USE_LLVM /usr/bin/llvm-config-19)" >> config.cmake
+RUN echo "set(HIDE_PRIVATE_SYMBOLS ON)" >> config.cmake
+RUN echo "set(USE_CUDA    OFF)" >> config.cmake
+RUN echo "set(USE_METAL   OFF)" >> config.cmake
+RUN echo "set(USE_VULKAN  OFF)" >> config.cmake
+RUN echo "set(USE_OPENCL  OFF)" >> config.cmake
+RUN echo "set(USE_CUBLAS  OFF)" >> config.cmake
+RUN echo "set(USE_CUDNN   OFF)" >> config.cmake
+RUN echo "set(USE_CUTLASS OFF)" >> config.cmake
+
+RUN /bin/bash -c 'cmake /root/tvm/ && cmake --build . -- -j$(nproc --all)'
+
+WORKDIR /root/
+RUN export TVM_LIBRARY_PATH=$TVM_BUILD_DIR
+RUN python3 -m pip install -e /root/tvm/python
 
 # ARG
 ARG TORCH_VERSION=2.0.1
 ARG TORCHVISION_VERSION=0.15.2
 ARG TFLite_VERSION=2.14.0
 ARG MXNET_VERSION=1.9.1
 ARG ONNX_VERSION=1.15.0
-ARG CAFFE_VERSION=1.1.0
 
 ARG PyTorch
 ARG TFLite
 ARG MXNet
 ARG ONNX
-ARG CAFFE
 
 RUN if [ "${PyTorch}" = "true"  ]; then \
     python3 -m pip install torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION}; \
@@ -65,14 +88,6 @@ RUN if [ "$ONNX" = "true"  ]; then \
     python3 -m pip install onnxruntime==${ONNX_VERSION} && python3 -m pip install onnx==${ONNX_VERSION}; \
     fi
 
-RUN if [ "$CAFFE" = "true"  ]; then \
-    /bin/bash -c "conda install -n tvm-env -y -c intel openvino-ie4py-ubuntu20=2022.1.0"; \
-    /bin/bash -c "conda install -n tvm-env -y -c intel scikit-image=0.17.2"; \
-    /bin/bash -c "conda install -n tvm-env -y -c intel caffe"; \
-    /bin/bash -c "conda install -n tvm-env -y -c anaconda libgcc-ng=11.2.0"; \
-    /bin/bash -c "conda install -n tvm-env -y -c defaults protobuf=3.17.2 libprotobuf=3.17.2 requests=2.27.1"; \
-    fi
-
 WORKDIR /tmp/open_model_zoo/
 RUN git remote add omz_custom_tvm https://github.com/itlab-vision/open_model_zoo_tvm.git && \ 
     git fetch omz_custom_tvm && \

requirements_frameworks.txt

@@ -17,6 +17,7 @@ dglgo==0.0.2
 
 tflite
 
+paddleslim==2.6.0
 paddlepaddle==2.6.0
 --extra-index-url https://mirror.baidu.com/pypi/simple