diff --git a/src/common/itt/include/openvino/itt.hpp b/src/common/itt/include/openvino/itt.hpp index c95f2be1f24a24..5c5e4061e61af8 100644 --- a/src/common/itt/include/openvino/itt.hpp +++ b/src/common/itt/include/openvino/itt.hpp @@ -44,9 +44,11 @@ namespace internal { domain_t domain(const char* name); handle_t handle(const char* name); void taskBegin(domain_t d, handle_t t); +void taskBegin(domain_t d, handle_t t, const char* key, uint64_t value); void taskEnd(domain_t d); void threadName(const char* name); void regionBegin(domain_t d, handle_t t); +void regionBegin(domain_t d, handle_t t, const char* key, uint64_t value); void regionEnd(domain_t d); } // namespace internal /** @@ -115,6 +117,27 @@ struct ScopedTask { internal::taskBegin(domain(), taskHandle); } + /** + * @brief Constructs a scoped task with an associated key-value metadata pair. + * + * Creates an ITT (Intel Tracing Technology) scoped task that will automatically + * begin upon construction and end when the object goes out of scope (RAII pattern). + * This overload allows attaching custom metadata to the task for enhanced profiling. + * + * @param taskHandle Handle identifying the task type for profiling tools. + * @param key String identifier for the metadata attribute (e.g., "InferenceID"). + * @param value Numeric value associated with the key (e.g., unique request ID). + * + * @note The task begins immediately upon construction via internal::taskBegin. + * @note The key string must remain valid for the duration of the task scope. + * @note This constructor is noexcept, ensuring exception safety in profiling code. + * + * @see ScopedTask(handle_t) for the basic constructor without metadata. + */ + ScopedTask(handle_t taskHandle, const char* key, uint64_t value) noexcept { + internal::taskBegin(domain(), taskHandle, key, value); + } + /** * @brief The ScopedTask destructor closes or ends the task scope */ @@ -143,6 +166,20 @@ struct ScopedRegion { internal::regionBegin(domain(), handle); } + /** + * @brief Constructs a scoped region with metadata for ITT profiling. + * + * @param handle Region identifier for profiling tools. + * @param key Metadata attribute name (e.g., "RequestID"). + * @param value Metadata numeric value associated with the key. + * + * @note Region begins immediately and ends when object goes out of scope (RAII). + * @note The key string must remain valid for the region's lifetime. + */ + ScopedRegion(handle_t handle, const char* key, uint64_t value) noexcept { + internal::regionBegin(domain(), handle, key, value); + } + /** * @brief The ScopedRegion destructor closes or ends the region scope */ @@ -308,8 +345,15 @@ class TaskChain { * @ingroup ov_dev_profiling * @brief Annotate section of code till scope exit for BASE/FULL modes regardless of profiling filter groups. * @details In case if handle or taskName absent, the current function name is used. + * @note All *_BASE() macros are enabled by default and will be used by supporting toolchains. The strings + * used/provided by these calls should follow the following rules: + * - Should NOT be deleted or modified to ensure correct visible names in profiling tools + * until CVS-179230 is implemented/resolved. + * - Should use string literals or constant strings when possible * @param domain [in] Known at compile time name of module or library (the domain name). * @param handleOrTaskName [in] The annotation name or handle for section of code. Parameter is optional. + * @param metadata_key [in] A metadata element's key as a string. Parameter is optional. + * @param metadata_value [in] The metadata value. Parameter is optional. */ #define OV_ITT_SCOPED_TASK_BASE(...) OV_PP_OVERLOAD(OV_ITT_SCOPED_TASK_BASE, __VA_ARGS__) @@ -321,6 +365,12 @@ class TaskChain { openvino::itt::ScopedTask OV_PP_CAT(ittScopedTask, __LINE__)( \ openvino::itt::handle(taskOrTaskName)); +#define OV_ITT_SCOPED_TASK_BASE_4(domain, taskOrTaskName, metadata_key, metadata_value) \ + openvino::itt::ScopedTask OV_PP_CAT(ittScopedTask, __LINE__)( \ + openvino::itt::handle(taskOrTaskName), \ + metadata_key, \ + metadata_value); + /** * @def OV_ITT_SCOPED_REGION_BASE(domain, handleOrRegionName) * @ingroup ov_dev_profiling @@ -328,8 +378,15 @@ class TaskChain { * @details In case if handle or regionName absent, the current function name is used. * @note Implements a region scope (single-active per thread; tasks started within * the region attach as children). + * @note All *_BASE() macros are enabled by default and will be used by supporting toolchains. The strings + * used/provided by these calls should follow the following rules: + * - Should NOT be deleted or modified to ensure correct visible names in profiling tools + * until CVS-179230 is implemented/resolved. + * - Should use string literals or constant strings when possible * @param domain [in] Known at compile time name of module or library (the domain name). * @param handleOrRegionName [in] The annotation name or handle for section of code. Parameter is optional. + * @param metadata_key [in] A metadata element's key as a string. Parameter is optional. + * @param metadata_value [in] The metadata value. Parameter is optional. */ #define OV_ITT_SCOPED_REGION_BASE(...) OV_PP_OVERLOAD(OV_ITT_SCOPED_REGION_BASE, __VA_ARGS__) @@ -341,6 +398,12 @@ class TaskChain { openvino::itt::ScopedRegion OV_PP_CAT(ittScopedRegion, __LINE__)( \ openvino::itt::handle(regionOrRegionName)); +#define OV_ITT_SCOPED_REGION_BASE_4(domain, regionOrRegionName, metadata_key, metadata_value) \ + openvino::itt::ScopedRegion OV_PP_CAT(ittScopedRegion, __LINE__)( \ + openvino::itt::handle(regionOrRegionName), \ + metadata_key, \ + metadata_value); + /** * @def OV_ITT_SCOPED_REGION(group, domain, handleOrRegionName) * @ingroup ov_dev_profiling diff --git a/src/common/itt/src/itt.cpp b/src/common/itt/src/itt.cpp index afc311d6b08eba..e5f6d7ef2eb4da 100644 --- a/src/common/itt/src/itt.cpp +++ b/src/common/itt/src/itt.cpp @@ -43,10 +43,16 @@ static thread_local uint64_t current_region_counter = 0; static thread_local void* current_region_handle = nullptr; domain_t domain(const char* name) { + if (!is_initialized() || name == nullptr) { + return nullptr; + } return reinterpret_cast(__itt_domain_create(name)); } handle_t handle(const char* name) { + if (!is_initialized() || name == nullptr) { + return nullptr; + } return reinterpret_cast(__itt_string_handle_create(name)); } @@ -64,6 +70,26 @@ void taskBegin(domain_t d, handle_t t) { } } +void taskBegin(domain_t d, handle_t t, const char* key, uint64_t value) { + if (!is_initialized() || d == nullptr || t == nullptr || key == nullptr) { + return; + } + if (!callStackDepth() || call_stack_depth++ < callStackDepth()) { + __itt_id parent_id = + current_region_counter != 0 ? __itt_id_make(current_region_handle, current_region_counter) : __itt_null; + __itt_domain* domain = reinterpret_cast<__itt_domain*>(d); + __itt_task_begin(domain, __itt_null, parent_id, reinterpret_cast<__itt_string_handle*>(t)); + // The task id to which the metadata is assigned to is not available at this point. It will + // default to the parent task's ID + __itt_metadata_add(domain, + __itt_null, + __itt_string_handle_create(key), + __itt_metadata_u64, + 1, + static_cast(const_cast(&value))); + } +} + void taskEnd(domain_t d) { if (!is_initialized() || d == nullptr) { return; @@ -94,6 +120,26 @@ void regionBegin(domain_t d, handle_t t) { reinterpret_cast<__itt_string_handle*>(t)); } +void regionBegin(domain_t d, handle_t t, const char* key, uint64_t value) { + if (!is_initialized() || d == nullptr || t == nullptr || key == nullptr) { + return; + } + std::lock_guard lock(region_mutex); + auto region_counter = nextRegionId(); + current_region_counter = region_counter; + current_region_handle = reinterpret_cast(t); + __itt_domain* domain = reinterpret_cast<__itt_domain*>(d); + __itt_id region_id = __itt_id_make(current_region_handle, current_region_counter); + __itt_region_begin(domain, region_id, __itt_null, reinterpret_cast<__itt_string_handle*>(t)); + // Associate the pair with the region + __itt_metadata_add(domain, + region_id, + __itt_string_handle_create(key), + __itt_metadata_u64, + 1, + static_cast(const_cast(&value))); +} + void regionEnd(domain_t d) { if (!is_initialized() || d == nullptr) { return; @@ -120,12 +166,16 @@ handle_t handle(const char*) { void taskBegin(domain_t, handle_t) {} +void taskBegin(domain_t, handle_t, const char*, uint64_t) {} + void taskEnd(domain_t) {} void threadName(const char*) {} void regionBegin(domain_t, handle_t) {} +void regionBegin(domain_t, handle_t, const char*, uint64_t) {} + void regionEnd(domain_t) {} #endif // ENABLE_PROFILING_ITT diff --git a/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp b/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp index 6faa16c09dc2ce..39f209b81e450b 100644 --- a/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp +++ b/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp @@ -193,6 +193,10 @@ class OPENVINO_RUNTIME_API IAsyncInferRequest : public IInferRequest { Pipeline m_pipeline; //!< Pipeline variable that should be filled by inherited class. Pipeline m_sync_pipeline; //!< Synchronous pipeline variable that should be filled by inherited class. + /** + * Tracks pipeline execution by having the same uniqueID for all stages of the same inference request. + */ + uint64_t m_infer_id; private: enum InferState { IDLE, BUSY, CANCELLED, STOP }; diff --git a/src/inference/dev_api/openvino/runtime/plugin_itt.hpp b/src/inference/dev_api/openvino/runtime/plugin_itt.hpp index e2713d61d9aa44..eb2cfec7b56e4f 100644 --- a/src/inference/dev_api/openvino/runtime/plugin_itt.hpp +++ b/src/inference/dev_api/openvino/runtime/plugin_itt.hpp @@ -16,6 +16,8 @@ namespace itt { namespace domains { OV_ITT_DOMAIN(Plugin) OV_ITT_DOMAIN(PluginLoadTime) +// Domain to define Inference phase tasks +OV_ITT_DOMAIN(Inference, "ov::phases::inference") } // namespace domains } // namespace itt } // namespace ov diff --git a/src/inference/src/cpp/core.cpp b/src/inference/src/cpp/core.cpp index 393752e8f37ec6..3142fd322218e8 100644 --- a/src/inference/src/cpp/core.cpp +++ b/src/inference/src/cpp/core.cpp @@ -79,7 +79,7 @@ std::map Core::get_versions(const std::string& device_name std::shared_ptr Core::read_model(const std::wstring& model_path, const std::wstring& bin_path, const ov::AnyMap& properties) const { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Read model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Read model"); OV_CORE_CALL_STATEMENT(return _impl->read_model(ov::util::wstring_to_string(model_path), ov::util::wstring_to_string(bin_path), properties);); @@ -89,24 +89,24 @@ std::shared_ptr Core::read_model(const std::wstring& model_path, std::shared_ptr Core::read_model(const std::string& model_path, const std::string& bin_path, const AnyMap& properties) const { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Read model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Read model"); OV_CORE_CALL_STATEMENT(return _impl->read_model(model_path, bin_path, properties);); } std::shared_ptr Core::read_model(const std::string& model, const ov::Tensor& weights) const { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Read model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Read model"); OV_CORE_CALL_STATEMENT(return _impl->read_model(model, weights);); } CompiledModel Core::compile_model(const std::shared_ptr& model, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); return compile_model(model, ov::default_device_name, config); } CompiledModel Core::compile_model(const std::shared_ptr& model, const std::string& device_name, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); OV_CORE_CALL_STATEMENT({ auto exec = _impl->compile_model(model, device_name, config); return {exec._ptr, exec._so}; @@ -114,19 +114,19 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, } CompiledModel Core::compile_model(const std::string& model_path, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); return compile_model(model_path, ov::default_device_name, config); } #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT CompiledModel Core::compile_model(const std::wstring& model_path, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); return compile_model(ov::util::wstring_to_string(model_path), config); } #endif CompiledModel Core::compile_model(const std::string& model_path, const std::string& device_name, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); OV_CORE_CALL_STATEMENT({ auto exec = _impl->compile_model(model_path, device_name, config); return {exec._ptr, exec._so}; @@ -137,7 +137,7 @@ CompiledModel Core::compile_model(const std::string& model_path, const std::stri CompiledModel Core::compile_model(const std::wstring& model_path, const std::string& device_name, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); return compile_model(ov::util::wstring_to_string(model_path), device_name, config); } #endif @@ -146,7 +146,7 @@ CompiledModel Core::compile_model(const std::string& model, const ov::Tensor& weights, const std::string& device_name, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); OV_CORE_CALL_STATEMENT({ auto exec = _impl->compile_model(model, weights, device_name, config); return {exec._ptr, exec._so}; @@ -156,7 +156,7 @@ CompiledModel Core::compile_model(const std::string& model, CompiledModel Core::compile_model(const std::shared_ptr& model, const RemoteContext& context, const AnyMap& config) { - OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::OV, "Compile model"); + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Phases, "Compile model"); OV_CORE_CALL_STATEMENT({ auto exec = _impl->compile_model(model, ov::SoPtr{context._impl, context._so}, config); return {exec._ptr, exec._so}; diff --git a/src/inference/src/dev/iasync_infer_request.cpp b/src/inference/src/dev/iasync_infer_request.cpp index c1b5395c972cf8..d2f0e3f66dbd9e 100644 --- a/src/inference/src/dev/iasync_infer_request.cpp +++ b/src/inference/src/dev/iasync_infer_request.cpp @@ -8,10 +8,14 @@ #include "openvino/runtime/isync_infer_request.hpp" #include "openvino/runtime/ivariable_state.hpp" +#include "openvino/runtime/plugin_itt.hpp" #include "openvino/runtime/threading/immediate_executor.hpp" #include "openvino/runtime/threading/istreams_executor.hpp" #include "openvino/runtime/variable_state.hpp" +/// @brief Thread-safe global counter for unique inference request IDs. +static std::atomic g_uid = {1}; + namespace { struct ImmediateStreamsExecutor : public ov::threading::ITaskExecutor { @@ -39,7 +43,8 @@ ov::IAsyncInferRequest::IAsyncInferRequest(const std::shared_ptr& const std::shared_ptr& callback_executor) : m_sync_request(request), m_request_executor(task_executor), - m_callback_executor(callback_executor) { + m_callback_executor(callback_executor), + m_infer_id(0) { if (m_request_executor && m_sync_request) m_pipeline = {{m_request_executor, [this] { m_sync_request->infer(); @@ -119,6 +124,7 @@ void ov::IAsyncInferRequest::start_async_thread_unsafe() { void ov::IAsyncInferRequest::run_first_stage(const Pipeline::iterator itBeginStage, const Pipeline::iterator itEndStage, const std::shared_ptr callbackExecutor) { + m_infer_id = g_uid++; auto& firstStageExecutor = std::get(*itBeginStage); OPENVINO_ASSERT(nullptr != firstStageExecutor); firstStageExecutor->run(make_next_stage_task(itBeginStage, itEndStage, std::move(callbackExecutor))); @@ -130,6 +136,11 @@ ov::threading::Task ov::IAsyncInferRequest::make_next_stage_task( const std::shared_ptr callbackExecutor) { return std::bind( [this, itStage, itEndStage](std::shared_ptr& callbackExecutor) mutable { + // Propagate the inference ID through all subsequent stages for this instance of the pipeline + OV_ITT_SCOPED_REGION_BASE(ov::itt::domains::Inference, + "Inference::pipeline", + "InferenceID", + m_infer_id); std::exception_ptr currentException = nullptr; auto& thisStage = *itStage; auto itNextStage = itStage + 1; diff --git a/src/inference/src/itt.hpp b/src/inference/src/itt.hpp index 9a65f95aa8a471..f2613b585aab70 100644 --- a/src/inference/src/itt.hpp +++ b/src/inference/src/itt.hpp @@ -18,6 +18,9 @@ namespace domains { OV_ITT_DOMAIN(OV, "ov"); OV_ITT_DOMAIN(ReadTime, "ov::ReadTime"); OV_ITT_DOMAIN(LoadTime, "ov::LoadTime"); +// Domain used for marking phases in the runtime +// DO NOT MODIFY OR DELETE! +OV_ITT_DOMAIN(Phases, "ov::phases"); } // namespace domains } // namespace itt } // namespace ov diff --git a/src/plugins/intel_cpu/src/infer_request.cpp b/src/plugins/intel_cpu/src/infer_request.cpp index b5b8dc91199ff1..772c5b44a0eaa1 100644 --- a/src/plugins/intel_cpu/src/infer_request.cpp +++ b/src/plugins/intel_cpu/src/infer_request.cpp @@ -66,7 +66,7 @@ SyncInferRequest::SyncInferRequest(CompiledModelHolder compiled_model) } void SyncInferRequest::create_infer_request() { - m_profiling_task = openvino::itt::handle("INTEL_CPU_INFER_" + m_compiled_model.name() + "_" + + m_profiling_task = openvino::itt::handle("SyncInferenceCPU::" + m_compiled_model.name() + ":" + std::to_string(m_compiled_model.id())); // Alocate memory for each tensor if static shape @@ -103,8 +103,7 @@ void SyncInferRequest::update_external_tensor_ptrs() { } void SyncInferRequest::infer() { - OV_ITT_SCOPED_REGION_BASE(itt::domains::ov_intel_cpu, - std::string("SyncInferenceRequest:") + m_compiled_model.name()); + OV_ITT_SCOPED_REGION_BASE(itt::domains::ov_intel_cpu, m_profiling_task); auto graphLock = m_compiled_model.lock(); auto&& graph = graphLock._graph; auto message = ov::threading::message_manager();