Move MiniCPM resampler to vision encoder (openvinotoolkit#1997)

Moving MiniCPM resampler to vision encoder allows to optimize
get_inputs_embeds() on second and subsequent chat iterations and not
re-run resampler each time.
Time measures are below for 6 chat iterations, where image is passed
only on the first iteration.

This branch:
encode time: 6585 ms
get_inputs_embeds time: 5 ms
Chat iteration 2:
encode time: 0 ms
get_inputs_embeds time: 3 ms
Chat iteration 3:
encode time: 0 ms
get_inputs_embeds time: 4 ms
Chat iteration 4:
encode time: 0 ms
get_inputs_embeds time: 4 ms
Chat iteration 5:
encode time: 0 ms
get_inputs_embeds time: 3 ms
Chat iteration 6:
encode time: 0 ms
get_inputs_embeds time: 3 ms

Master:
Chat iteration 1:
encode time: 6176 ms
get_inputs_embeds time: 358 ms
Chat iteration 2:
encode time: 0 ms
get_inputs_embeds time: 338 ms
Chat iteration 3:
encode time: 0 ms
get_inputs_embeds time: 339 ms
Chat iteration 4:
encode time: 0 ms
get_inputs_embeds time: 340 ms
Chat iteration 5:
encode time: 0 ms
get_inputs_embeds time: 335 ms
Chat iteration 6:
encode time: 0 ms
get_inputs_embeds time: 337 ms

Author: popovaan

src/cpp/include/openvino/genai/common_types.hpp

@@ -0,0 +1,18 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <map>
+#include "openvino/core/core.hpp"
+
+namespace ov {
+namespace genai {
+
+/// @brief A map of models for VLMPipeline constructor. 
+/// Key is model name (e.g. "vision_embeddings", "text_embeddings", "language", "resampler")
+/// and value is a pair of model IR as string and weights as tensor.
+using ModelsMap = std::map<std::string, std::pair<std::string, ov::Tensor>>;
+
+}
+}

src/cpp/include/openvino/genai/llm_pipeline.hpp

@@ -14,6 +14,7 @@
 #include "openvino/genai/streamer_base.hpp"
 #include "openvino/genai/perf_metrics.hpp"
 #include "openvino/genai/scheduler_config.hpp"
+#include "openvino/genai/common_types.hpp"
 
 namespace ov {
 namespace genai {
@@ -26,11 +27,6 @@ using OptionalGenerationConfig = std::optional<GenerationConfig>;
 using EncodedInputs = std::variant<ov::Tensor, TokenizedInputs>;
 using StringInputs = std::variant<std::string, std::vector<std::string>>;
 
-/// @brief A map of models for VLMPipeline constructor. 
-/// Key is model name (e.g. "vision_embeddings", "text_embeddings", "language", "resampler")
-/// and value is a pair of model IR as string and weights as tensor.
-using ModelsMap = std::map<std::string, std::pair<std::string, ov::Tensor>>;
-
 /**
 * @brief Structure to store resulting batched tokens and scores for each batch sequence.
 * The first num_return_sequences elements correspond to the first batch element.

src/cpp/src/visual_language/inputs_embedder.cpp

@@ -88,8 +88,7 @@ InputsEmbedder::IInputsEmbedder::IInputsEmbedder(
         const ov::AnyMap device_config) :
     m_vlm_config{vlm_config},
     m_vision_encoder(VisionEncoder::create(
-        utils::get_model_weights_pair(models_map, "vision_embeddings").first,
-        utils::get_model_weights_pair(models_map, "vision_embeddings").second,
+        models_map,
         config_dir_path,
         m_vlm_config.model_type,
         device,

src/cpp/src/visual_language/minicpm/classes.cpp

@@ -412,7 +412,31 @@ EncodedImage VisionEncoderMiniCPM::encode(const ov::Tensor& image, const ov::Any
     ctx_clip.image_size = config.image_size;
     std::copy(config.norm_mean.begin(), config.norm_mean.end(), ctx_clip.image_mean);
     std::copy(config.norm_std.begin(), config.norm_std.end(), ctx_clip.image_std);
-    return llava_image_embed_make_with_bytes_slice(ctx_clip, image, encoder, config.max_slice_nums, config.scale_resolution, config.patch_size, 0 == config.max_slice_nums);
+    EncodedImage encoded_image = llava_image_embed_make_with_bytes_slice(ctx_clip, image, encoder, config.max_slice_nums, config.scale_resolution, config.patch_size, 0 == config.max_slice_nums);
+    encoded_image.resampled_image = resample_encoded_image(encoded_image);
+    return encoded_image;
+}
+
+ResampledImage VisionEncoderMiniCPM::resample_encoded_image(const EncodedImage& encoded_image) {
+    const ov::Tensor& resampled_source = resample(encoded_image.resized_source, {encoded_image.resized_source_size});
+    std::vector<std::vector<ov::Tensor>> vision_embed_tensors;
+    if (encoded_image.slices) {
+        size_t token_idx = 0;
+        const ov::Shape& slices_shape = encoded_image.slices.get_shape();
+        vision_embed_tensors.resize(slices_shape.at(0));
+        for (size_t i = 0; i < slices_shape.at(0); ++i) {
+            std::vector<ov::Tensor> vision_embeds;
+            vision_embeds.resize(slices_shape.at(1));
+            for (size_t ja = 0; ja < slices_shape.at(1); ++ja) {
+                size_t d2 = slices_shape.at(2);
+                size_t d3 = slices_shape.at(3);
+                ov::Tensor encoded_view{ov::element::f32, {1, d2, d3}, encoded_image.slices.data<float>() + (i * slices_shape.at(1) + ja) * d2 * d3};
+                vision_embeds[ja] = resample(encoded_view, {encoded_image.slices_size});
+            }
+            vision_embed_tensors[i] = vision_embeds;
+        }
+    }
+    return {resampled_source, vision_embed_tensors};
 }
 
 namespace {
@@ -542,44 +566,6 @@ void adjust_pos_cache(
 
 } // namespace
 
-InputsEmbedderMiniCPM::InputsEmbedderMiniCPM(
-    const VLMConfig& vlm_config,
-    const std::filesystem::path& model_dir,
-    const std::string& device,
-    const ov::AnyMap device_config) :
-    IInputsEmbedder(vlm_config, model_dir, device, device_config) {
-    auto compiled_model =
-        utils::singleton_core().compile_model(model_dir / "openvino_resampler_model.xml", device, device_config);
-    ov::genai::utils::print_compiled_model_properties(compiled_model, "VLM resampler model");
-    m_ireq_queue_resampler = std::make_unique<CircularBufferQueue<ov::InferRequest>>(
-        compiled_model.get_property(ov::optimal_number_of_infer_requests),
-        [&compiled_model]() -> ov::InferRequest {
-            return compiled_model.create_infer_request();
-        });
-    m_pos_embed_cache = get_2d_sincos_pos_embed(m_vlm_config.hidden_size, {70, 70});
-}
-
-InputsEmbedderMiniCPM::InputsEmbedderMiniCPM(
-    const VLMConfig& vlm_config,
-    const ModelsMap& models_map,
-    const Tokenizer& tokenizer,
-    const std::filesystem::path& config_dir_path,
-    const std::string& device,
-    const ov::AnyMap device_config) :
-    IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) {
-    auto compiled_model = utils::singleton_core().compile_model(
-        utils::get_model_weights_pair(models_map, "resampler").first,
-        utils::get_model_weights_pair(models_map, "resampler").second,
-        device,
-        device_config);
-    m_ireq_queue_resampler = std::make_unique<CircularBufferQueue<ov::InferRequest>>(
-        compiled_model.get_property(ov::optimal_number_of_infer_requests),
-        [&compiled_model]() -> ov::InferRequest {
-            return compiled_model.create_infer_request();
-        });
-    m_pos_embed_cache = get_2d_sincos_pos_embed(m_vlm_config.hidden_size, {70, 70});
-}
-
 ov::Tensor InputsEmbedderMiniCPM::get_inputs_embeds(const std::string& prompt, const std::vector<ov::genai::EncodedImage>& images, ov::genai::VLMPerfMetrics& metrics) {
     auto [unified_prompt, images_sequence] = normalize_prompt(
         prompt,
@@ -648,7 +634,7 @@ ov::Tensor InputsEmbedderMiniCPM::get_inputs_embeds(const std::string& prompt, c
     float* inputs_embeds_data = inputs_embeds.data<float>();
     for (size_t image_id : images_sequence) {
         const EncodedImage& encoded_image = images.at(image_id - m_prev_image_id);
-        const ov::Tensor& resampled_source = resample(encoded_image.resized_source, {encoded_image.resized_source_size});
+        const ov::Tensor& resampled_source = encoded_image.resampled_image.resampled_source;
         auto emb = resampled_source.data<float>();
         ids = std::find(ids, end, im_start_id);
         OPENVINO_ASSERT(end != ids);
@@ -660,10 +646,7 @@ ov::Tensor InputsEmbedderMiniCPM::get_inputs_embeds(const std::string& prompt, c
             const ov::Shape& slices_shape = encoded_image.slices.get_shape();
             for (size_t i = 0; i < slices_shape.at(0); ++i) {
                 for (size_t ja = 0; ja < slices_shape.at(1); ++ja) {
-                    size_t d2 = slices_shape.at(2);
-                    size_t d3 = slices_shape.at(3);
-                    ov::Tensor encoded_view{ov::element::f32, {1, d2, d3}, encoded_image.slices.data<float>() + (i * slices_shape.at(1) + ja) * d2 * d3};
-                    const ov::Tensor& vision_embed_tensor_i_j = resample(encoded_view, {encoded_image.slices_size});
+                    const ov::Tensor& vision_embed_tensor_i_j = encoded_image.resampled_image.vision_embed_tensors[i][ja];
                     ids = std::find(ids, end, slice_start_id);
                     OPENVINO_ASSERT(end != ids);
                     ++ids;
@@ -703,7 +686,7 @@ bool InputsEmbedderMiniCPM::prompt_has_image_tag(const std::string& prompt) cons
     return IInputsEmbedder::prompt_has_image_tag(prompt) || prompt.find(NATIVE_TAG) != std::string::npos;
 }
 
-ov::Tensor InputsEmbedderMiniCPM::resample(const ov::Tensor& encoded_image, const std::vector<ImageSize>& target_sizes) {
+ov::Tensor VisionEncoderMiniCPM::resample(const ov::Tensor& encoded_image, const std::vector<ImageSize>& target_sizes) {
     size_t bs = encoded_image.get_shape().at(0);
     std::vector<size_t> patch_len{target_sizes.size()};
     std::transform(target_sizes.begin(), target_sizes.end(), patch_len.begin(), [](const ImageSize& height_width) {
@@ -747,7 +730,62 @@ ov::Tensor InputsEmbedderMiniCPM::resample(const ov::Tensor& encoded_image, cons
     resampler.set_tensor("pos_embed", pos_embed);  // [H*W, N, new_hidden_size]
     resampler.set_tensor("key_padding_mask", key_padding_mask);  // [N, H*W]
     resampler.infer();
-    return resampler.get_output_tensor();  // [N, query_num, new_hidden_size]
+    auto resampler_out = resampler.get_output_tensor();
+    // resampler_out is bound to infer request and the data may become corrupted after next resampler inference 
+    // so we need to return a copy to make sure data does not get corrupted 
+    ov::Tensor res(resampler_out.get_element_type(), resampler_out.get_shape());
+    std::memcpy(res.data(), resampler_out.data(), resampler_out.get_byte_size());
+    return res;  // [N, query_num, new_hidden_size]
+}
+
+VisionEncoderMiniCPM::VisionEncoderMiniCPM(
+        const std::filesystem::path& model_dir,
+        const std::string& device,
+        const ov::AnyMap properties) : VisionEncoder{model_dir, device, properties}  {
+    m_vlm_config = utils::from_config_json_if_exists<VLMConfig>(model_dir, "config.json");
+    auto compiled_model = utils::singleton_core().compile_model(model_dir / "openvino_resampler_model.xml", device, properties);
+    ov::genai::utils::print_compiled_model_properties(compiled_model, "VLM resampler model");
+    m_ireq_queue_resampler = std::make_unique<CircularBufferQueue<ov::InferRequest>>(
+        compiled_model.get_property(ov::optimal_number_of_infer_requests),
+        [&compiled_model]() -> ov::InferRequest {
+            return compiled_model.create_infer_request();
+        }); 
+    m_pos_embed_cache = get_2d_sincos_pos_embed(m_vlm_config.hidden_size, {70, 70});
+}
+
+VisionEncoderMiniCPM::VisionEncoderMiniCPM(
+        const ModelsMap& models_map,
+        const std::filesystem::path& config_dir_path,
+        const std::string& device,
+        const ov::AnyMap device_config) : VisionEncoder{models_map, config_dir_path, device, device_config} {
+    const auto& resampler_model = utils::get_model_weights_pair(models_map, "resampler").first;
+    const auto& resampler_weights = utils::get_model_weights_pair(models_map, "resampler").second;
+    m_vlm_config = utils::from_config_json_if_exists<VLMConfig>(config_dir_path, "config.json");
+    auto compiled_model = utils::singleton_core().compile_model(resampler_model, resampler_weights, device, device_config);
+    ov::genai::utils::print_compiled_model_properties(compiled_model, "VLM resampler model");
+    m_ireq_queue_resampler = std::make_unique<CircularBufferQueue<ov::InferRequest>>(
+        compiled_model.get_property(ov::optimal_number_of_infer_requests),
+        [&compiled_model]() -> ov::InferRequest {
+            return compiled_model.create_infer_request();
+        }); 
+    m_pos_embed_cache = get_2d_sincos_pos_embed(m_vlm_config.hidden_size, {70, 70});
 }
 
+
+InputsEmbedderMiniCPM::InputsEmbedderMiniCPM(
+    const VLMConfig& vlm_config,
+    const std::filesystem::path& model_dir,
+    const std::string& device,
+    const ov::AnyMap device_config) :
+    IInputsEmbedder(vlm_config, model_dir, device, device_config) {}
+
+InputsEmbedderMiniCPM::InputsEmbedderMiniCPM(
+    const VLMConfig& vlm_config,
+    const ModelsMap& models_map,
+    const Tokenizer& tokenizer,
+    const std::filesystem::path& config_dir_path,
+    const std::string& device,
+    const ov::AnyMap device_config) :
+    IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) {}
+
 } // namespace ov::genai

src/cpp/src/visual_language/minicpm/classes.hpp

@@ -13,13 +13,6 @@
 namespace ov::genai {
 
 class VisionEncoderMiniCPM : public VisionEncoder {
-public:
-    using VisionEncoder::VisionEncoder;
-
-    EncodedImage encode(const ov::Tensor& image, const ov::AnyMap& config_map) override;
-};
-
-class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
     // A resampler model to resample image embeddings.
     // [N, H*W, old_hidden_size] is the input shape.
     // [N, query_num, hidden_size] is the output shape.
@@ -28,6 +21,27 @@ class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
     // [70, 70, hidden_size]. 70 is the initial guess of the image
     // height and width after dividing by patch_size.
     ov::Tensor m_pos_embed_cache;
+    // VLM config
+    VLMConfig m_vlm_config;
+
+    ov::Tensor resample(const ov::Tensor& encoded_image, const std::vector<ImageSize>& target_sizes);
+public:
+    VisionEncoderMiniCPM(
+        const std::filesystem::path& model_dir,
+        const std::string& device,
+        const ov::AnyMap properties);
+
+
+    VisionEncoderMiniCPM(
+        const ModelsMap& models_map,
+        const std::filesystem::path& config_dir_path,
+        const std::string& device,
+        const ov::AnyMap device_config);
+    EncodedImage encode(const ov::Tensor& image, const ov::AnyMap& config_map) override;
+    ResampledImage resample_encoded_image(const EncodedImage& image);
+};
+
+class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
     size_t m_prev_image_id = 0;
 
 public:
@@ -36,7 +50,7 @@ class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
         const std::filesystem::path& model_dir,
         const std::string& device,
         const ov::AnyMap device_config);
-
+    
     InputsEmbedderMiniCPM(
         const VLMConfig& vlm_config,
         const ModelsMap& models_map,
@@ -55,8 +69,6 @@ class InputsEmbedderMiniCPM : public InputsEmbedder::IInputsEmbedder {
 
     bool prompt_has_image_tag(const std::string& prompt) const override;
 
-private:
-    ov::Tensor resample(const ov::Tensor& encoded_image, const std::vector<ImageSize>& target_sizes);
 };
 
 } // namespace ov::genai

src/cpp/src/visual_language/vision_encoder.cpp

@@ -26,12 +26,13 @@ VisionEncoder::VisionEncoder(const std::filesystem::path& model_dir, const std::
 }
 
 VisionEncoder::VisionEncoder(
-    const std::string& model,
-    const ov::Tensor& weights,
+    const ModelsMap& models_map,
     const std::filesystem::path& config_dir_path,
     const std::string& device,
     const ov::AnyMap device_config) {
-    auto compiled_model = utils::singleton_core().compile_model(model, weights, device, device_config);
+    const auto& vision_encoder_model = utils::get_model_weights_pair(models_map, "vision_embeddings").first;
+    const auto& vision_encoder_weights = utils::get_model_weights_pair(models_map, "vision_embeddings").second;
+    auto compiled_model = utils::singleton_core().compile_model(vision_encoder_model, vision_encoder_weights, device, device_config);
     ov::genai::utils::print_compiled_model_properties(compiled_model, "VLM vision embeddings model");
     m_ireq_queue_vision_encoder = std::make_unique<CircularBufferQueue<ov::InferRequest>>(
         compiled_model.get_property(ov::optimal_number_of_infer_requests),
@@ -64,24 +65,23 @@ VisionEncoder::Ptr VisionEncoder::create(const std::filesystem::path& model_dir,
 }
 
 VisionEncoder::Ptr VisionEncoder::create(
-    const std::string& model,
-    const ov::Tensor& weights,
+    const ModelsMap& models_map,
     const std::filesystem::path& config_dir_path,
     const VLMModelType model_type,
     const std::string& device,
     const ov::AnyMap device_config) {
     if (model_type == VLMModelType::MINICPM) {
-        return std::make_shared<VisionEncoderMiniCPM>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderMiniCPM>(models_map, config_dir_path, device, device_config);
     } else if (model_type == VLMModelType::LLAVA) {
-        return std::make_shared<VisionEncoderLLaVA>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderLLaVA>(models_map, config_dir_path, device, device_config);
     } else if (model_type == VLMModelType::LLAVA_NEXT) {
-        return std::make_shared<VisionEncoderLLaVANext>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderLLaVANext>(models_map, config_dir_path, device, device_config);
     } else if (model_type == VLMModelType::INTERNVL_CHAT) {
-        return std::make_shared<VisionEncoderInternVLChat>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderInternVLChat>(models_map, config_dir_path, device, device_config);
     } else if (model_type == VLMModelType::PHI3_V) {
-        return std::make_shared<VisionEncoderPhi3V>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderPhi3V>(models_map, config_dir_path, device, device_config);
     } else if (model_type == VLMModelType::QWEN2_VL) {
-        return std::make_shared<VisionEncoderQwen2VL>(model, weights, config_dir_path, device, device_config);
+        return std::make_shared<VisionEncoderQwen2VL>(models_map, config_dir_path, device, device_config);
     } else {
         OPENVINO_THROW("Unsupported model type in VLM VisionEncoder class. Please, create feature request on new model support");
     }

src/cpp/src/visual_language/vision_encoder.hpp

@@ -5,6 +5,7 @@
 #include <memory>
 #include "openvino/runtime/infer_request.hpp"
 
+#include "openvino/genai/common_types.hpp"
 #include "visual_language/vlm_config.hpp"
 #include "visual_language/processor_config.hpp"
 #include "circular_buffer_queue.hpp"
@@ -18,6 +19,12 @@ struct ImageSize {
     size_t width;
 };
 
+
+struct ResampledImage {
+    ov::Tensor resampled_source;
+    std::vector<std::vector<ov::Tensor>> vision_embed_tensors;
+};
+
 /// @brief Embeddings of a given image. The number of slices is no
 /// greater than ProcessorConfig's max_slice_nums.
 struct EncodedImage {
@@ -45,6 +52,9 @@ struct EncodedImage {
 
     /// @brief Original size of the image
     ImageSize original_image_size;
+
+    /// @brief Resampled image, used only by MiniCPM.
+    ResampledImage resampled_image;
 };
 
 /// @brief A class used to infer embeddings of an image using
@@ -67,16 +77,14 @@ class VisionEncoder {
         const ov::AnyMap properties = {});
 
     /// @brief Constructs the encoder from models map.
-    /// @param model Model IR as string (openvino_vision_embeddings_model.xml)
-    /// @param weights Model weights as tensor (openvino_vision_embeddings_model.bin)
+    /// @param models_map Models map
     /// @param config_dir_path A path to directory containing preprocessor_config.json.
     /// @param model_type A type of VLM model.
     /// @param device A device to compile the encoder for.
     /// @param properties A config to be passed to
     /// ov::Core::compile_model().
     static VisionEncoder::Ptr create(
-        const std::string& model,
-        const ov::Tensor& weights,
+        const ModelsMap& models_map,
         const std::filesystem::path& config_dir_path,
         const VLMModelType model_type,
         const std::string& device,
@@ -110,8 +118,7 @@ class VisionEncoder {
         const ov::AnyMap properties);
 
     VisionEncoder(
-        const std::string& model,
-        const ov::Tensor& weights,
+        const ModelsMap& models_map,
         const std::filesystem::path& config_dir_path,
         const std::string& device,
         const ov::AnyMap properties);