Optimize get_inputs_embeds() for Qwen2VL. (openvinotoolkit#2037)

popovaan · ilya-lavrenov · Wovchena · web-flow · commit fb0c87072522 · 2025-04-21T12:22:04.000+04:00
Image embeddings merger is moved to a separate method and in chat mode
it is used only when new images are passed.
Time measures are below for 6 chat iterations for Qwen2-VL-2B-Instruct,
where image is passed on the first and third iteration.

This branch:
Chat iteration 1 (new image):
encode time: 2012 ms
get_inputs_embeds time: 7683 ms
Chat iteration 2:
encode time: 0 ms
get_inputs_embeds time: 7 ms
Chat iteration 3 (new image):
encode time: 2359 ms
get_inputs_embeds time: 29179 ms
Chat iteration 4:
encode time: 0 ms
get_inputs_embeds time: 10 ms
Chat iteration 5:
encode time: 0 ms
get_inputs_embeds time: 11 ms
Chat iteration 6:
encode time: 0 ms
get_inputs_embeds time: 8 ms



Master:
Chat iteration 1 (new image):
encode time: 1893ms
get_inputs_embeds time: 8394ms
Chat iteration 2:
encode time: 0ms
get_inputs_embeds time: 7664ms
Chat iteration 3 (new image):
encode time: 2126ms
get_inputs_embeds time: 27954ms
Chat iteration 4:
encode time: 0ms
get_inputs_embeds time: 27944ms
Chat iteration 5:
encode time: 0ms
get_inputs_embeds time: 27974ms
Chat iteration 6:
encode time: 0ms
get_inputs_embeds time: 27970ms

---------

Co-authored-by: Ilya Lavrenov &lt;ilya.lavrenov@intel.com&gt;
Co-authored-by: Vladimir Zlobin &lt;vladimir.zlobin@intel.com&gt;
diff --git a/src/cpp/src/icontinuous_batching.cpp b/src/cpp/src/icontinuous_batching.cpp
@@ -174,13 +174,17 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
             prompt_with_tags = add_image_tags_to_prompt(prompt_with_tags, rgbs, m_history_images.size());
         }
         m_history.push_back({{"role", "user"}, {"content", prompt_with_tags}});
+        auto start_get_inputs_embeds = std::chrono::steady_clock::now();
         const auto encoded_images = m_inputs_embedder->encode_images(rgbs);
         m_history_images.insert(m_history_images.end(), encoded_images.begin(), encoded_images.end());
-        std::string templated_history = m_tokenizer.apply_chat_template(m_history, true);
 
+        std::string templated_history = m_tokenizer.apply_chat_template(m_history, true);
         m_inputs_embedder->set_apply_chat_template_status(false);
 
-        input_embeds_list.push_back(m_inputs_embedder->get_inputs_embeds(templated_history, m_history_images, vlm_perf_metrics[0]));
+        input_embeds_list.push_back(m_inputs_embedder->get_inputs_embeds(templated_history, m_history_images, vlm_perf_metrics[0], rgbs.size() > 0));
+        auto end_get_inputs_embeds = std::chrono::steady_clock::now();
+        vlm_perf_metrics[0].vlm_raw_metrics.prepare_embeddings_durations.emplace_back(PerfMetrics::get_microsec(end_get_inputs_embeds - start_get_inputs_embeds));
+
     } else {
         for (size_t i = 0; i < prompts.size(); i++) {
             const auto& prompt = prompts[i];
diff --git a/src/cpp/src/visual_language/inputs_embedder.cpp b/src/cpp/src/visual_language/inputs_embedder.cpp
@@ -244,8 +244,8 @@ ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const st
     return m_impl->get_inputs_embeds(prompt, images, metrics);
 }
 
-ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
-    return m_impl->get_inputs_embeds(prompt, images, metrics);
+ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
+    return m_impl->get_inputs_embeds(prompt, images, metrics, recalculate_merged_embeddings);
 }
 
 std::vector<ov::genai::EncodedImage> InputsEmbedder::encode_images(const std::vector<ov::Tensor>& images) {
diff --git a/src/cpp/src/visual_language/inputs_embedder.hpp b/src/cpp/src/visual_language/inputs_embedder.hpp
@@ -37,7 +37,7 @@ class InputsEmbedder {
     // compute input embedding for prompt and multiple images
     ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::Tensor>& images, ov::genai::VLMPerfMetrics& metrics);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics);
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true);
 
     std::vector<ov::genai::EncodedImage> encode_images(const std::vector<ov::Tensor>& images);
 
@@ -98,7 +98,7 @@ class InputsEmbedder {
         size_t m_image_id = 0;
 
     public:
-        virtual ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) = 0;
+        virtual ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) = 0;
 
         ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::Tensor>& images, ov::genai::VLMPerfMetrics& metrics);
 
diff --git a/src/cpp/src/visual_language/internvl_chat/classes.cpp b/src/cpp/src/visual_language/internvl_chat/classes.cpp
@@ -226,7 +226,7 @@ InputsEmbedderInternVLChat::InputsEmbedderInternVLChat(
     const ov::AnyMap device_config) :
     IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) { }
 
-ov::Tensor InputsEmbedderInternVLChat::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderInternVLChat::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     std::string image_start_token = m_vlm_config.image_start_token;
     std::string image_context_token = m_vlm_config.image_context_token;
     std::string image_end_token = m_vlm_config.image_end_token;
diff --git a/src/cpp/src/visual_language/internvl_chat/classes.hpp b/src/cpp/src/visual_language/internvl_chat/classes.hpp
@@ -35,7 +35,7 @@ class InputsEmbedderInternVLChat : public InputsEmbedder::IInputsEmbedder {
         const std::string& device,
         const ov::AnyMap device_config);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
 };
 
 } // namespace ov::genai
diff --git a/src/cpp/src/visual_language/llava/classes.cpp b/src/cpp/src/visual_language/llava/classes.cpp
@@ -114,7 +114,7 @@ std::vector<ov::genai::EncodedImage> InputsEmbedderLLaVA::encode_images(const st
     return embeds;
 }
 
-ov::Tensor InputsEmbedderLLaVA::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderLLaVA::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     std::string image_token = m_vlm_config.im_start;
 
     std::string formatted_prompt;
diff --git a/src/cpp/src/visual_language/llava/classes.hpp b/src/cpp/src/visual_language/llava/classes.hpp
@@ -35,7 +35,7 @@ class InputsEmbedderLLaVA : public InputsEmbedder::IInputsEmbedder {
         const std::string& device,
         const ov::AnyMap device_config);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
 
     std::vector<ov::genai::EncodedImage> encode_images(const std::vector<ov::Tensor>& images) override;
 protected:
diff --git a/src/cpp/src/visual_language/llava_next/classes.cpp b/src/cpp/src/visual_language/llava_next/classes.cpp
@@ -343,7 +343,7 @@ std::vector<ov::genai::EncodedImage> InputsEmbedderLLaVANext::encode_images(cons
     return embeds;
 }
 
-ov::Tensor InputsEmbedderLLaVANext::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderLLaVANext::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     std::string image_token = m_vlm_config.im_start;
 
     std::string formatted_prompt;
diff --git a/src/cpp/src/visual_language/llava_next/classes.hpp b/src/cpp/src/visual_language/llava_next/classes.hpp
@@ -22,7 +22,7 @@ class InputsEmbedderLLaVANext : public InputsEmbedderLLaVA {
 public:
     using InputsEmbedderLLaVA::InputsEmbedderLLaVA;
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
 
     std::vector<ov::genai::EncodedImage> encode_images(const std::vector<ov::Tensor>& images) override;
 };
diff --git a/src/cpp/src/visual_language/minicpm/classes.cpp b/src/cpp/src/visual_language/minicpm/classes.cpp
@@ -566,7 +566,7 @@ void adjust_pos_cache(
 
 } // namespace
 
-ov::Tensor InputsEmbedderMiniCPM::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderMiniCPM::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     auto [unified_prompt, images_sequence] = normalize_prompt(
         prompt,
         NATIVE_TAG,
diff --git a/src/cpp/src/visual_language/minicpm/classes.hpp b/src/cpp/src/visual_language/minicpm/classes.hpp
@@ -59,8 +59,7 @@ class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
         const std::string& device,
         const ov::AnyMap device_config);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
-
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
     void update_chat_history(const std::string& decoded_results, const ov::genai::GenerationStatus generation_finish_status) override;
 
     void start_chat(const std::string& system_message) override;
diff --git a/src/cpp/src/visual_language/phi3_vision/classes.cpp b/src/cpp/src/visual_language/phi3_vision/classes.cpp
@@ -662,7 +662,7 @@ InputsEmbedderPhi3V::InputsEmbedderPhi3V(
     const ov::AnyMap device_config) :
     IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) {}
 
-ov::Tensor InputsEmbedderPhi3V::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderPhi3V::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     size_t base_id = m_tokens_per_images.size();
     std::string image_prompt = normalize_prompt(prompt, base_id, images.size());
     std::vector<ov::Tensor> images_features_proj;
diff --git a/src/cpp/src/visual_language/phi3_vision/classes.hpp b/src/cpp/src/visual_language/phi3_vision/classes.hpp
@@ -48,7 +48,7 @@ class InputsEmbedderPhi3V : public InputsEmbedder::IInputsEmbedder {
         const std::string& device,
         const ov::AnyMap device_config);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
 
     void update_chat_history(const std::string& decoded_results, const ov::genai::GenerationStatus generation_finish_status) override;
 
diff --git a/src/cpp/src/visual_language/qwen2vl/classes.cpp b/src/cpp/src/visual_language/qwen2vl/classes.cpp
@@ -281,25 +281,18 @@ InputsEmbedderQwen2VL::InputsEmbedderQwen2VL(
         });
 }
 
-ov::Tensor InputsEmbedderQwen2VL::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
+ov::Tensor InputsEmbedderQwen2VL::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {
     auto [unified_prompt, images_sequence] = normalize_prompt(prompt, NATIVE_TAG, NATIVE_TAG, m_image_id, images.size());
-    std::vector<ov::Tensor> image_embeds;
     std::vector<std::array<size_t, 3>> images_grid_thw;
-    image_embeds.reserve(images.size());
     images_grid_thw.reserve(images.size());
     
     for (const auto& encoded_image : images) {
-        ov::Tensor single_image_embeds = encoded_image.resized_source;
-        image_embeds.push_back(std::move(single_image_embeds));
-
         size_t grid_t = 1;
         size_t grid_h = encoded_image.resized_source_size.height;
         size_t grid_w = encoded_image.resized_source_size.width;
         images_grid_thw.push_back({grid_t, grid_h, grid_w});
     }
 
-    std::vector<ov::Tensor> reordered_image_embeds;
-    std::vector<std::array<size_t, 3>> reordered_images_grid_thw;
     for (size_t new_image_id : images_sequence) {
         auto [grid_t, grid_h, grid_w] = images_grid_thw.at(new_image_id - m_image_id);
         size_t merge_length = std::pow(m_vision_encoder->get_processor_config().merge_size, 2);
@@ -311,10 +304,7 @@ ov::Tensor InputsEmbedderQwen2VL::get_inputs_embeds(const std::string& prompt, c
         }
         expanded_tag += m_vlm_config.vision_end_token;
         unified_prompt.replace(unified_prompt.find(NATIVE_TAG), NATIVE_TAG.length(), expanded_tag);
-        reordered_image_embeds.push_back(image_embeds.at(new_image_id - m_image_id));
-        reordered_images_grid_thw.push_back(images_grid_thw.at(new_image_id - m_image_id));
     }
-    m_image_id = images_sequence.empty() ? m_image_id : *std::max_element(images_sequence.begin(), images_sequence.end()) + 1;
 
     ov::Tensor input_ids = get_encoded_input_ids(unified_prompt, metrics);
     CircularBufferQueueElementGuard<EmbeddingsRequest> embeddings_request_guard(m_embedding->get_request_queue().get());
@@ -335,16 +325,24 @@ ov::Tensor InputsEmbedderQwen2VL::get_inputs_embeds(const std::string& prompt, c
     int64_t position_ids_max_element = *std::max_element(m_position_ids.data<int64_t>(), m_position_ids.data<int64_t>() + m_position_ids.get_size());
     m_rope_delta = position_ids_max_element + 1 - static_cast<int64_t>(input_ids.get_shape().at(1));
 
-    if (!m_is_chat_conversation) {
-        m_image_id = 0;
-    }
     if (images.empty()) {
         ov::Tensor inputs_embeds(text_embeds.get_element_type(), text_embeds.get_shape());
         std::memcpy(inputs_embeds.data(), text_embeds.data(), text_embeds.get_byte_size());
         return inputs_embeds;
     }
+    ov::Tensor merged_image_embeddings_tensor;
+    if (recalculate_merged_embeddings) {
+        m_merged_image_embeddings = run_image_embeddings_merger(images, images_sequence, m_image_id, m_vlm_config);
+    }
+    merged_image_embeddings_tensor = m_merged_image_embeddings;
 
-    return merge_text_and_image_embeddings_qwen2vl(input_ids, text_embeds, reordered_image_embeds, reordered_images_grid_thw, image_pad_token_id);
+    if (m_is_chat_conversation) {
+        m_image_id = images_sequence.empty() ? m_image_id : *std::max_element(images_sequence.begin(), images_sequence.end()) + 1;
+    }
+    else {
+        m_image_id = 0;
+    }
+    return merge_text_and_image_embeddings_qwen2vl(input_ids, text_embeds, merged_image_embeddings_tensor, image_pad_token_id);
 }
 
 std::pair<ov::Tensor, std::optional<int64_t>> InputsEmbedderQwen2VL::get_position_ids(const size_t inputs_embeds_size, const size_t history_size) {
@@ -376,17 +374,34 @@ bool InputsEmbedderQwen2VL::prompt_has_image_tag(const std::string& prompt) cons
     return IInputsEmbedder::prompt_has_image_tag(prompt) || prompt.find(NATIVE_TAG) != std::string::npos;
 }
 
-ov::Tensor InputsEmbedderQwen2VL::merge_text_and_image_embeddings_qwen2vl(
-    const ov::Tensor& input_ids,
-    const ov::Tensor& text_embeds,
-    const std::vector<ov::Tensor>& image_embeds,
-    const std::vector<std::array<size_t, 3>> images_grid_thw,
-    const int64_t image_pad_token_id) {
+ov::Tensor InputsEmbedderQwen2VL::run_image_embeddings_merger(const std::vector<EncodedImage>& images, const std::vector<size_t>& images_sequence, size_t image_id, const VLMConfig& vlm_config) {
+    std::vector<ov::Tensor> image_embeds;
+    std::vector<std::array<size_t, 3>> images_grid_thw;
+    image_embeds.reserve(images.size());
+    images_grid_thw.reserve(images.size());
+    
+    for (const auto& encoded_image : images) {
+        ov::Tensor single_image_embeds = encoded_image.resized_source;
+        image_embeds.push_back(std::move(single_image_embeds));
+
+        size_t grid_t = 1;
+        size_t grid_h = encoded_image.resized_source_size.height;
+        size_t grid_w = encoded_image.resized_source_size.width;
+        images_grid_thw.push_back({grid_t, grid_h, grid_w});
+    }
+
+    std::vector<ov::Tensor> reordered_image_embeds;
+    std::vector<std::array<size_t, 3>> reordered_images_grid_thw;
+    for (size_t new_image_id : images_sequence) {
+        reordered_image_embeds.push_back(image_embeds.at(new_image_id - image_id));
+        reordered_images_grid_thw.push_back(images_grid_thw.at(new_image_id - image_id));
+    }
+
     // Calculate cumulative sequence lengths for attention mask
     std::vector<int32_t> cu_seqlens;
     cu_seqlens.push_back(0);
     int32_t cumsum = 0;
-    for (const auto& grid_thw : images_grid_thw) {
+    for (const auto& grid_thw : reordered_images_grid_thw) {
         size_t slice_len = grid_thw.at(1) * grid_thw.at(2);
         for (size_t t = 0; t < grid_thw.at(0); ++t) {
             cumsum += slice_len;
@@ -412,28 +427,27 @@ ov::Tensor InputsEmbedderQwen2VL::merge_text_and_image_embeddings_qwen2vl(
 
     // Concatenate image embeddings 
     ov::Tensor concatenated_images;
-    if (image_embeds.size() == 1) {
-        concatenated_images = image_embeds.at(0);
+    if (reordered_image_embeds.size() == 1) {
+        concatenated_images = reordered_image_embeds.at(0);
     } else {
         size_t total_length = 0;
-        for (const auto& embed : image_embeds) {
+        for (const auto& embed : reordered_image_embeds) {
             total_length += embed.get_shape().at(0);
         }
-        size_t hidden_dim = image_embeds.at(0).get_shape().at(1);
+        size_t hidden_dim = reordered_image_embeds.at(0).get_shape().at(1);
         
-        concatenated_images = ov::Tensor(image_embeds.at(0).get_element_type(), {total_length, hidden_dim});
+        concatenated_images = ov::Tensor(reordered_image_embeds.at(0).get_element_type(), {total_length, hidden_dim});
         float* concat_data = concatenated_images.data<float>();
         
         size_t offset = 0;
-        for (const auto& embed : image_embeds) {
+        for (const auto& embed : reordered_image_embeds) {
             size_t embed_size = embed.get_shape().at(0) * embed.get_shape().at(1);
             std::memcpy(concat_data + offset, embed.data(), embed.get_byte_size());
             offset += embed_size;
         }
     }
 
-    ov::Tensor rotary_pos_emb = get_rotary_pos_emb(images_grid_thw);
-
+    ov::Tensor rotary_pos_emb = get_rotary_pos_emb(reordered_images_grid_thw);
     CircularBufferQueueElementGuard<ov::InferRequest> infer_request_guard(this->m_ireq_queue_vision_embeddings_merger.get());
     ov::InferRequest& vision_embeddings_merger = infer_request_guard.get();
     vision_embeddings_merger.set_tensor("hidden_states", concatenated_images);
@@ -442,6 +456,16 @@ ov::Tensor InputsEmbedderQwen2VL::merge_text_and_image_embeddings_qwen2vl(
     vision_embeddings_merger.infer();
     ov::Tensor processed_vision_embeds = vision_embeddings_merger.get_output_tensor();
 
+    ov::Tensor res = ov::Tensor(processed_vision_embeds.get_element_type(), processed_vision_embeds.get_shape());
+    std::memcpy(res.data(), processed_vision_embeds.data(), processed_vision_embeds.get_byte_size());
+    return res;
+}
+
+ov::Tensor InputsEmbedderQwen2VL::merge_text_and_image_embeddings_qwen2vl(
+    const ov::Tensor& input_ids,
+    const ov::Tensor& text_embeds, 
+    const ov::Tensor& processed_vision_embeds,
+    const int64_t image_pad_token_id) {
     ov::Tensor merged_embeds(text_embeds.get_element_type(), text_embeds.get_shape());
     std::memcpy(merged_embeds.data(), text_embeds.data(), text_embeds.get_byte_size());
 
diff --git a/src/cpp/src/visual_language/qwen2vl/classes.hpp b/src/cpp/src/visual_language/qwen2vl/classes.hpp
@@ -30,6 +30,13 @@ class InputsEmbedderQwen2VL : public InputsEmbedder::IInputsEmbedder {
 
     ov::Tensor m_position_ids;
     int64_t m_rope_delta = 0;
+    ov::Tensor m_merged_image_embeddings;
+
+    ov::Tensor run_image_embeddings_merger(
+        const std::vector<EncodedImage>& images, 
+        const std::vector<size_t>& images_sequence, 
+        size_t image_id, 
+        const VLMConfig& vlm_config);
 
 public:
     InputsEmbedderQwen2VL(
@@ -46,7 +53,7 @@ class InputsEmbedderQwen2VL : public InputsEmbedder::IInputsEmbedder {
         const std::string& device,
         const ov::AnyMap device_config);
 
-    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) override;
+    ov::Tensor get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings = true) override;
 
     std::pair<ov::Tensor, std::optional<int64_t>> get_position_ids(const size_t inputs_embeds_size, const size_t history_size) override;
 
@@ -59,9 +66,8 @@ class InputsEmbedderQwen2VL : public InputsEmbedder::IInputsEmbedder {
 protected:
     ov::Tensor merge_text_and_image_embeddings_qwen2vl(
         const ov::Tensor& input_ids,
-        const ov::Tensor& text_embeds,
-        const std::vector<ov::Tensor>& image_embeds,
-        const std::vector<std::array<size_t, 3>> images_grid_thw,
+        const ov::Tensor& text_embeds, 
+        const ov::Tensor& merged_image_embeds,
         const int64_t image_pad_token_id
     );
 

Original file line number	Diff line number	Diff line change
`@@ -244,8 +244,8 @@ ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const st`
`244`	`244`	`return m_impl->get_inputs_embeds(prompt, images, metrics);`
`245`	`245`	`}`
`246`	`246`
`247`		`-ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {`
`248`		`- return m_impl->get_inputs_embeds(prompt, images, metrics);`
	`247`	`+ov::Tensor InputsEmbedder::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {`
	`248`	`+ return m_impl->get_inputs_embeds(prompt, images, metrics, recalculate_merged_embeddings);`
`249`	`249`	`}`
`250`	`250`
`251`	`251`	`std::vector<ov::genai::EncodedImage> InputsEmbedder::encode_images(const std::vector<ov::Tensor>& images) {`
Original file line number	Diff line number	Diff line change
`@@ -114,7 +114,7 @@ std::vector<ov::genai::EncodedImage> InputsEmbedderLLaVA::encode_images(const st`
`114`	`114`	`return embeds;`
`115`	`115`	`}`
`116`	`116`
`117`		`-ov::Tensor InputsEmbedderLLaVA::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {`
	`117`	`+ov::Tensor InputsEmbedderLLaVA::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {`
`118`	`118`	`std::string image_token = m_vlm_config.im_start;`
`119`	`119`
`120`	`120`	`std::string formatted_prompt;`
Original file line number	Diff line number	Diff line change
`@@ -343,7 +343,7 @@ std::vector<ov::genai::EncodedImage> InputsEmbedderLLaVANext::encode_images(cons`
`343`	`343`	`return embeds;`
`344`	`344`	`}`
`345`	`345`
`346`		`-ov::Tensor InputsEmbedderLLaVANext::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {`
	`346`	`+ov::Tensor InputsEmbedderLLaVANext::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics, bool recalculate_merged_embeddings) {`
`347`	`347`	`std::string image_token = m_vlm_config.im_start;`
`348`	`348`
`349`	`349`	`std::string formatted_prompt;`