[Protopipe] Add support to set workload type at runtime

src/plugins/intel_npu/tools/opencv_version.json

@@ -1,3 +1,3 @@
 {
-    "opencv" : "dee619b628e86d187bbe7a55b5fe6c7bfbb8b64d"
+    "opencv" : "c1c893ff73581567b6d1a847e8dafc9c052d907c"
 }

src/plugins/intel_npu/tools/protopipe/README.md

@@ -265,7 +265,14 @@ Every stream has the following execution parameters:
 - `target_fps` - **Optional**. Execution frequency of the stream. `target_fps = 1000 / frames_interval_in_ms`. `target_fps` and `frames_interval_in_ms` are mutually exclusive and cannot be provided together.
 - `target_latency_in_ms` - **Optional**. When iteration isn't finished within specified interval, the next frame will be dropped from execution. (**Default**: Disabled)
 - `op_desc`/`conections` or `network` - **Required**. Execution graph structure. Follow [Graph structure](#graph-structure) for the details.
-
+- `workload_type` - **Optional**. Configure workload type updates at runtime. `workload_type` has the following parameters:
+  - `initial_value` - **Required**. Initial value for workload type. This value will be used when starting the simulation.
+  - `change_to` - **Required**. A list containing the values used to update the workload type during runtime.
+  - `change_interval` - **Required**. A value that represents the number of seconds or iterations depending on stream configuration (iteration_count or exec_time_in_secs is set). The workload type will be update according to this interval value.
+  - `repeat` - **Optional**. If this value is set to true and all the values in the `change_to` list were used, but the simulation is still running, the values in the `change_to` list will be reiterated. If set to false or not set at all, workload type updates will stop after the last value in `change_to` list, even if the simulation is still running.
+  - #### example:
+    `workload_type : {initial_value: Default, change_to: [Efficient, Default],  change_interval: 5, repeat: true}`
+    - This config sets the initial value of workload_type to Default, then after every 5 seconds/iterations it changes to Efficient then to Default again. Considering that `exec_time_in_secs` is set to 20, and `repeat` is true, there will be 4 changes: Default->Efficient after 5 seconds, Efficient->Default after 10 seconds, Default->Efficient after 15 seconds and Efficient->Default after 20 seconds. If `repeat` is false there will be only 2 changes: after 5 seconds Default->Efficient and after 10 seconds Efficient->Default, for the remaining 10 seconds of the simulation workload_type will remain set to Default. `workload_type` values should start with capital letter.
 ### Config example
 Consider the following scenario that consists of two parallel streams specified on `config.yaml`:  
 ```

src/plugins/intel_npu/tools/protopipe/main.cpp

@@ -5,16 +5,17 @@
 
 #include <future>
 #include <iostream>
+#include <memory>
 #include <regex>
 
 #include <gflags/gflags.h>
 
 #include "parser/parser.hpp"
-#include "scenario/scenario_graph.hpp"
 #include "simulation/performance_mode.hpp"
 #include "simulation/reference_mode.hpp"
 #include "simulation/validation_mode.hpp"
 #include "simulation/accuracy_mode.hpp"
+#include "simulation/workload_type.hpp"
 
 #include "utils/error.hpp"
 #include "utils/logger.hpp"
@@ -259,8 +260,29 @@ int main(int argc, char* argv[]) {
                     }
                     criterion = global_criterion->clone();
                 }
+                std::shared_ptr<WorkloadTypeInfo> workload_type;
+                if (stream.workload_type.has_value())
+                {
+                    workload_type = std::make_shared<WorkloadTypeInfo>();
+                    workload_type->wl_ov =  std::make_shared<cv::gapi::wip::ov::WorkloadTypeOV>();
+                    workload_type->wl_onnx= std::make_shared<cv::gapi::onnx::WorkloadTypeONNX>();
+                    workload_type->workload_config = stream.workload_type.value();
+                    criterion->setWorkloadTrigger(workload_type);
+                }
+
                 auto simulation = createSimulation(FLAGS_mode, std::move(stream), FLAGS_inference_only, config);
+                if (workload_type)
+                    simulation->workload = workload_type;
+
+
                 auto compiled = compileSimulation(simulation, FLAGS_pipeline, FLAGS_drop_frames);
+
+                if (simulation->workload) {
+                    simulation->workload->wl_onnx->set(simulation->workload->workload_config.initial_value);
+                    simulation->workload->wl_ov->set(simulation->workload->workload_config.initial_value);
+                    LOG_INFO() << "Setting initial value of workload type to " << simulation->workload->workload_config.initial_value << std::endl;
+                }
+
                 tasks.emplace_back(std::move(compiled), std::move(stream_name), std::move(criterion));
                 runner.add(std::ref(tasks.back()));
             }

src/plugins/intel_npu/tools/protopipe/src/parser/config.cpp

@@ -4,12 +4,15 @@
 //
 
 #include "parser/config.hpp"
+#include "scenario/criterion.hpp"
 
 #include "utils/error.hpp"
 #include "utils/logger.hpp"
 
+#include <cstdint>
 #include <filesystem>
 #include <map>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -649,6 +652,35 @@ static InferenceParams adjustParams(InferenceParams&& params, const GlobalOption
     return adjustParams(std::get<ONNXRTParams>(std::move(params)), opts);
 }
 
+static void parseWorkloadType(const YAML::Node& node, StreamDesc& stream) {
+    WorkloadTypeDesc workload_type;
+    if (node["initial_value"]) {
+        workload_type.initial_value = node["initial_value"].as<std::string>();
+    }
+    else {
+        THROW_ERROR("Missing workload type initial value");
+    }
+    if (node["change_interval"]) {
+        workload_type.change_interval = node["change_interval"].as<uint64_t>();
+    }
+    else {
+        THROW_ERROR("Missing workload type change interval");
+    }
+    if (node["change_to"]) {
+        workload_type.changes = node["change_to"].as<std::vector<std::string>>();
+    }
+    else {
+        THROW_ERROR("Missing workload type change values");
+    }
+    if (node["repeat"]) {
+        workload_type.repeat = node["repeat"].as<bool>();
+    }
+    else {
+        workload_type.repeat = false;
+    }
+    stream.workload_type = std::make_optional(workload_type);
+}
+
 static StreamDesc parseStream(const YAML::Node& node, const GlobalOptions& opts, const std::string& default_name,
                               const ReplaceBy& replace_by) {
     StreamDesc stream;
@@ -681,6 +713,10 @@ static StreamDesc parseStream(const YAML::Node& node, const GlobalOptions& opts,
         const auto iteration_count = node["iteration_count"].as<uint64_t>();
         stream.criterion = std::make_shared<Iterations>(iteration_count);
     }
+    if (node["workload_type"])
+    {
+        parseWorkloadType(node["workload_type"], stream);
+    }
 
     auto networks_list = parseNetworks(node["network"]);
     const auto delay_in_us = node["delay_in_us"] ? node["delay_in_us"].as<uint32_t>() : 0u;
@@ -807,7 +843,10 @@ static StreamDesc parseAdvancedStream(const YAML::Node& node, const GlobalOption
         const auto iteration_count = node["iteration_count"].as<uint64_t>();
         stream.criterion = std::make_shared<Iterations>(iteration_count);
     }
-
+    if (node["workload_type"])
+    {
+        parseWorkloadType(node["workload_type"], stream);
+    }
     auto op_descs = node["op_desc"].as<std::vector<OpDesc>>();
     std::vector<std::vector<std::string>> connections;
     if (node["connections"]) {

src/plugins/intel_npu/tools/protopipe/src/parser/parser.cpp

@@ -6,8 +6,6 @@
 #include "parser/parser.hpp"
 #include "parser/config.hpp"
 
-#include "utils/error.hpp"
-
 #include <yaml-cpp/yaml.h>
 
 ScenarioParser::ScenarioParser(const std::string& filepath): m_filepath(filepath) {

src/plugins/intel_npu/tools/protopipe/src/parser/parser.hpp

@@ -6,27 +6,37 @@
 #pragma once
 
 #include <filesystem>
+#include <memory>
 #include <string>
 #include <vector>
 
-#include "scenario/criterion.hpp"
 #include "scenario/inference.hpp"
 #include "scenario/scenario_graph.hpp"
 
+struct ITermCriterion;
+using ITermCriterionPtr = std::shared_ptr<ITermCriterion>;
+
+struct WorkloadTypeDesc {
+    std::string initial_value;
+    std::vector<std::string> changes;
+    uint64_t change_interval;
+    bool repeat;
+};
 struct StreamDesc {
     // NB: Commons parameters for all modes
     std::string name;
     uint64_t frames_interval_in_us;
     ScenarioGraph graph;
     InferenceParamsMap infer_params_map;
-    ITermCriterion::Ptr criterion;
+    ITermCriterionPtr criterion;
     // Mode specific params
     ModelsAttrMap<IAccuracyMetric::Ptr> metrics_map;
     ModelsAttrMap<IRandomGenerator::Ptr> initializers_map;
     ModelsAttrMap<std::string> input_data_map;
     ModelsAttrMap<std::string> output_data_map;
     std::optional<double> target_latency;
     std::optional<std::filesystem::path> per_iter_outputs_path;
+    std::optional<WorkloadTypeDesc> workload_type;
 };
 
 struct ScenarioDesc {

src/plugins/intel_npu/tools/protopipe/src/scenario/criterion.cpp

@@ -6,16 +6,46 @@
 #include "criterion.hpp"
 
 #include <chrono>
+#include <cstdint>
+#include <ostream>
+#include "utils/logger.hpp"
 
 #include "utils/utils.hpp"
 
+void ITermCriterion::setWorkloadTrigger(std::shared_ptr<WorkloadTypeInfo> workload_ptr) {
+    workload_type = workload_ptr;
+}
+void ITermCriterion::checkWorkloadTrigger() {
+    if (workload_type) {
+        if (customCheck(workload_type->workload_config.change_interval)) {
+            // update based on changes
+            uint64_t next_index = workload_index % workload_type->workload_config.changes.size();
+            if (!workload_type->workload_config.repeat &&
+                workload_index >= workload_type->workload_config.changes.size()) {
+                return;
+            }
+            std::string next_value = workload_type->workload_config.changes[next_index];
+            workload_index++;
+            LOG_INFO() << "Update workload type to " << next_value << " after " << workload_index * workload_type->workload_config.change_interval << " seconds/iterations" << std::endl;
+
+            workload_type->wl_onnx->set(next_value);
+            workload_type->wl_ov->set(next_value);
+        }
+    }
+}
+
+
 Iterations::Iterations(uint64_t num_iters): m_num_iters(num_iters), m_counter(0) {
 }
 
 bool Iterations::check() const {
     return m_counter != m_num_iters;
 }
 
+bool Iterations::customCheck(uint64_t value) {
+    return m_counter != 0 && m_counter % value == 0;
+}
+
 void Iterations::update() {
     ++m_counter;
 }
@@ -35,6 +65,15 @@ bool TimeOut::check() const {
     return utils::timestamp<std::chrono::microseconds>() - m_start_ts < m_time_in_us;
 }
 
+bool TimeOut::customCheck(uint64_t value) {
+    auto now = utils::timestamp<std::chrono::microseconds>() - m_start_ts;
+    if (now - last_update >= (value * 1'000'000)) {
+        last_update = now;
+        return true;
+    }
+    return false;
+}
+
 void TimeOut::update(){/* do nothing */};
 
 void TimeOut::init() {
@@ -57,6 +96,10 @@ bool CombinedCriterion::check() const {
     return m_lhs->check() && m_rhs->check();
 }
 
+bool CombinedCriterion::customCheck(uint64_t value) {
+    return m_lhs->customCheck(value) && m_rhs->customCheck(value);
+}
+
 void CombinedCriterion::update() {
     m_lhs->update();
     m_rhs->update();

src/plugins/intel_npu/tools/protopipe/src/scenario/criterion.hpp

@@ -8,12 +8,18 @@
 #include <cstdint>
 #include <memory>
 
+#include "simulation/workload_type.hpp"
 struct ITermCriterion {
     using Ptr = std::shared_ptr<ITermCriterion>;
     virtual void init() = 0;
     virtual void update() = 0;
     virtual bool check() const = 0;
+    virtual bool customCheck(uint64_t value) = 0;
     virtual ITermCriterion::Ptr clone() const = 0;
+    void setWorkloadTrigger(std::shared_ptr<WorkloadTypeInfo>);
+    void checkWorkloadTrigger();
+    std::shared_ptr<WorkloadTypeInfo> workload_type;
+    uint64_t workload_index = 0;
 };
 
 class Iterations : public ITermCriterion {
@@ -23,6 +29,7 @@ class Iterations : public ITermCriterion {
     void init() override;
     void update() override;
     bool check() const override;
+    bool customCheck(uint64_t value) override;
     ITermCriterion::Ptr clone() const override;
 
 private:
@@ -37,11 +44,14 @@ class TimeOut : public ITermCriterion {
     void init() override;
     void update() override;
     bool check() const override;
+    bool customCheck(uint64_t value) override;
+
     ITermCriterion::Ptr clone() const override;
 
 private:
     uint64_t m_time_in_us;
     uint64_t m_start_ts;
+    uint64_t last_update = 0;
 };
 
 class CombinedCriterion : public ITermCriterion {
@@ -52,6 +62,7 @@ class CombinedCriterion : public ITermCriterion {
     void init() override;
     void update() override;
     bool check() const override;
+    bool customCheck(uint64_t value) override;
     ITermCriterion::Ptr clone() const override;
 
 private:

src/plugins/intel_npu/tools/protopipe/src/simulation/executor.cpp

@@ -26,6 +26,7 @@ PipelinedExecutor::Output PipelinedExecutor::runLoop(cv::GRunArgs&& inputs, Call
     const auto start_tick = clock_t::now();
     m_compiled.start();
     while (criterion->check()) {
+        criterion->checkWorkloadTrigger();
         if (!callback(m_compiled)) {
             break;
         }
@@ -52,6 +53,7 @@ SyncExecutor::Output SyncExecutor::runLoop(Callback callback, ITermCriterion::Pt
     const auto start_tick = clock_t::now();
     criterion->init();
     while (criterion->check()) {
+        criterion->checkWorkloadTrigger();
         if (!callback(m_compiled)) {
             break;
         }

src/plugins/intel_npu/tools/protopipe/src/simulation/performance_mode.cpp

@@ -294,6 +294,14 @@ std::shared_ptr<PipelinedCompiled> PerformanceSimulation::compilePipelined(Dummy
         // TODO: Extend also for ONNXRT backend
         compile_args += cv::compile_args(cv::gapi::wip::ov::benchmark_mode{});
     }
+    if (workload) {
+        if (workload->wl_onnx) {
+            compile_args += cv::compile_args(workload->wl_onnx);
+        }
+        if (workload->wl_ov) {
+            compile_args += cv::compile_args(workload->wl_ov);
+        }
+    }
     auto compiled = m_comp.compileStreaming(descr_of(sources), std::move(compile_args));
     return std::make_shared<PipelinedSimulation>(std::move(compiled), std::move(sources), m_comp.getOutMeta().size());
 }
@@ -304,7 +312,14 @@ std::shared_ptr<SyncCompiled> PerformanceSimulation::compileSync(const bool drop
         // TODO: Extend also for ONNXRT backend
         compile_args += cv::compile_args(cv::gapi::wip::ov::benchmark_mode{});
     }
-
+    if (workload) {
+        if (workload->wl_onnx) {
+            compile_args += cv::compile_args(workload->wl_onnx);
+        }
+        if (workload->wl_ov) {
+            compile_args += cv::compile_args(workload->wl_ov);
+        }
+    }
     const uint32_t max_parallel_branches = m_comp.getMaxParallelBranches();
     if (max_parallel_branches > 1u) {
         LOG_INFO() << "Found at most " << max_parallel_branches