Kcz/add mp4 spliting into frames

README.md

@@ -157,6 +157,13 @@ image_data = ov.Tensor(image_data)
 
 prompt = "Can you describe the image?"
 result = pipe.generate(prompt, image=image_data, max_new_tokens=100)
+
+# To input multiple images, use 'images='
+# result = pipe.generate(prompt, images=[image_data], max_new_tokens=100)
+
+# To input videos frames, use 'videos=', frames_data layout = [Frame num, H, W, C]
+# result = pipe.generate(prompt, videos=[frames_data], max_new_tokens=100)
+
 print(result.texts[0])
 ```
 
@@ -178,6 +185,12 @@ int main(int argc, char* argv[]) {
         ov::genai::image(rgb),
         ov::genai::max_new_tokens(100)
     ) << '\n';
+
+    // To input multiple images, use 'images'
+    // pipe.generate(prompt, ov::genai::images(std::vector<ov::Tensor>{rgb}), ov::genai::max_new_tokens(100));
+
+    // To input videos frames, use 'videos'
+    // pipe.generate(prompt, ov::genai::videos(std::vector<ov::Tensor>{frames}), ov::genai::max_new_tokens(100));
 }
 ```
 

src/cpp/include/openvino/genai/visual_language/pipeline.hpp

@@ -127,7 +127,7 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
     /// If the prompt doesn't contain image or video tags, but images or videos are
     /// provided, the tags are prepended to the prompt.
     /// @param images Image to be prepended to a prompt.
-    /// @param videos Videos to be prepended to a prompt.
+    /// @param videos Multiple videos, each providing multiple frames, to be prepended to a prompt.
     /// @param generation_config A config to follow for text generation.
     /// @param streamer A streamer to acquire intermediate result.
     /// @return A string generated by a model.
@@ -291,8 +291,8 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
 /*
  * utils that allow to use generate() in the following way:
  * pipe.generate(prompt, ov::genai::image(image_tensor)).
- * pipe.generate(prompt, ov::genai::images(video_tensor)).
- * pipe.generate(prompt, ov::genai::videos(video_tensor)).
+ * pipe.generate(prompt, ov::genai::images(image_tensors)).
+ * pipe.generate(prompt, ov::genai::videos(videos_tensors)).
 */
 static constexpr ov::Property<ov::Tensor> image{"image"};
 static constexpr ov::Property<std::vector<ov::Tensor>> images{"images"};

src/cpp/src/continuous_batching/pipeline.cpp

@@ -276,13 +276,12 @@ std::vector<VLMDecodedResults> ContinuousBatchingPipeline::generate(
 std::vector<VLMDecodedResults> ContinuousBatchingPipeline::generate(
     const std::vector<std::string>& prompts,
     const std::vector<std::vector<ov::Tensor>>& images,
-    const std::vector<std::vector<ov::Tensor>>& video,
+    const std::vector<std::vector<ov::Tensor>>& videos,
     const std::vector<GenerationConfig>& sampling_params,
     const StreamerVariant& streamer) {
-    return m_impl->generate(prompts, images, video, sampling_params, streamer);
+    return m_impl->generate(prompts, images, videos, sampling_params, streamer);
 }
 
-
 void ContinuousBatchingPipeline::start_chat(const std::string& system_message) {
     m_impl->finish_chat();
     m_impl->start_chat(system_message);

src/cpp/src/continuous_batching/pipeline_base.cpp

@@ -38,6 +38,7 @@ void ContinuousBatchingPipeline::IContinuousBatchingPipeline::finish_chat() {
     m_history_videos.clear();
     m_history_image_ids.clear();
     m_history_video_ids.clear();
+    m_history_vision_count.clear();
     if (m_inputs_embedder) {
         m_inputs_embedder->finish_chat();
     }
@@ -164,52 +165,65 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
              const std::vector<std::vector<ov::Tensor>>& images_vector,
              const std::vector<std::vector<ov::Tensor>>& videos_vector,
              const std::vector<GenerationConfig>& sampling_params,
-             const StreamerVariant& streamer)  {
+             const StreamerVariant& streamer) {
     auto generate_start_time = std::chrono::steady_clock::now();
     OPENVINO_ASSERT(m_model_input_type == ModelInputType::EMBEDDINGS);
 
     OPENVINO_ASSERT(prompts.size() == sampling_params.size(), "Number of prompts should be equal to the number of generation configs.");
-    OPENVINO_ASSERT(prompts.size() == images_vector.size() && prompts.size() == videos_vector.size(), "Number of prompts should be equal to the number of images or video vectors.");
+    if (images_vector.size() > 0)
+        OPENVINO_ASSERT(prompts.size() == images_vector.size(), "Number of prompts should be equal to the number of images vectors.");
+    if (videos_vector.size() > 0)
+        OPENVINO_ASSERT(prompts.size() == videos_vector.size(), "Number of prompts should be equal to the number of videos vectors.");
 
     std::vector<ov::Tensor> input_embeds_list;
     std::vector<ov::Tensor> token_type_ids_list;
 
     std::vector<VLMPerfMetrics> vlm_perf_metrics(prompts.size());
     std::vector<EncodedImage> encoded_images = {};
     std::vector<EncodedVideo> encoded_videos = {};
+    bool recalculate_merged_embeddings = images_vector.size() > 0 || videos_vector.size() > 0;
 
     if (m_is_chat_conversation) {
         OPENVINO_ASSERT(1 == prompts.size(), "Can't chat with multiple prompts");
         const auto& prompt = prompts[0];
         auto start_get_inputs_embeds = std::chrono::steady_clock::now();
 
-        encoded_images = m_inputs_embedder->encode_images(images_vector[0]);
+        encoded_images = m_inputs_embedder->encode_images(images_vector.size() > 0 ? images_vector[0] : std::vector<ov::Tensor>{});
         m_history_images.insert(m_history_images.end(), encoded_images.begin(), encoded_images.end());
 
-        encoded_videos = m_inputs_embedder->encode_videos(videos_vector[0]);
+        encoded_videos = m_inputs_embedder->encode_videos(videos_vector.size() > 0 ? videos_vector[0] : std::vector<ov::Tensor>{});
         m_history_videos.insert(m_history_videos.end(), encoded_videos.begin(), encoded_videos.end());
 
         auto [unified_prompt, image_sequence, video_sequence] = m_inputs_embedder->normalize_prompt(prompt, m_image_id, m_video_id, encoded_images, encoded_videos);
 
         m_history.push_back({{"role", "user"}, {"content", unified_prompt}});
         m_history_image_ids.insert(m_history_image_ids.end(), image_sequence.begin(), image_sequence.end());
         m_history_video_ids.insert(m_history_video_ids.end(), video_sequence.begin(), video_sequence.end());
+        m_history_vision_count.emplace_back(std::make_pair(video_sequence.size(), image_sequence.size()));
 
         std::string templated_history = m_tokenizer.apply_chat_template(m_history, true);
 
         m_inputs_embedder->set_apply_chat_template_status(false);
         if (m_inputs_embedder->has_token_type_ids()) {
-            auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(templated_history, m_history_images, vlm_perf_metrics[0], images_vector.size() > 0, m_history_image_ids);
+            auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(templated_history,
+                                                                                             m_history_images,
+                                                                                             m_history_videos,
+                                                                                             vlm_perf_metrics[0],
+                                                                                             recalculate_merged_embeddings,
+                                                                                             m_history_image_ids,
+                                                                                             m_history_video_ids,
+                                                                                             m_history_vision_count);
             input_embeds_list.push_back(std::move(embeds));
             token_type_ids_list.push_back(std::move(tt_ids));
         } else {
             input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(templated_history,
-                                                                    m_history_images,
-                                                                    m_history_videos,
-                                                                    vlm_perf_metrics[0],
-                                                                    true,
-                                                                    m_history_image_ids,
-                                                                    m_history_video_ids));
+                                                                                m_history_images,
+                                                                                m_history_videos,
+                                                                                vlm_perf_metrics[0],
+                                                                                recalculate_merged_embeddings,
+                                                                                m_history_image_ids,
+                                                                                m_history_video_ids,
+                                                                                m_history_vision_count));
         }
 
         auto end_get_inputs_embeds = std::chrono::steady_clock::now();
@@ -230,11 +244,17 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
             m_inputs_embedder->set_apply_chat_template_status(sampling_params[i].apply_chat_template);
 
             if (m_inputs_embedder->has_token_type_ids()) {
-                auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(unified_prompt, encoded_images, vlm_perf_metrics[i], true, image_sequence);
+                auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(unified_prompt,
+                                                                                                 encoded_images,
+                                                                                                 encoded_videos,
+                                                                                                 vlm_perf_metrics[i],
+                                                                                                 recalculate_merged_embeddings,
+                                                                                                 image_sequence,
+                                                                                                 video_sequence);
                 input_embeds_list.push_back(std::move(embeds));
                 token_type_ids_list.push_back(std::move(tt_ids));
             } else {
-                input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(unified_prompt, encoded_images, encoded_videos, vlm_perf_metrics[i], true, image_sequence, video_sequence));
+                input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(unified_prompt, encoded_images, encoded_videos, vlm_perf_metrics[i], recalculate_merged_embeddings, image_sequence, video_sequence));
             }
 
             auto end_get_inputs_embeds = std::chrono::steady_clock::now();
@@ -278,6 +298,11 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
                 m_history_image_ids.pop_back();
                 m_history_images.pop_back();
             }
+            for (size_t idx = 0; idx < encoded_videos.size(); idx++) {
+                m_history_video_ids.pop_back();
+                m_history_videos.pop_back();
+            }
+            m_history_vision_count.pop_back();
         }
     }
     return results;
@@ -307,12 +332,13 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::add_request(uint64_t re
     return add_request(request_id, inputs, sampling_params, token_type_ids);
 }
 
-GenerationHandle 
-ContinuousBatchingPipeline::IContinuousBatchingPipeline::add_request(uint64_t request_id,
-                                        const std::string& prompt,
-                                        const std::vector<ov::Tensor>& images,
-                                        const std::vector<ov::Tensor>& videos,
-                                        GenerationConfig sampling_params) {
+GenerationHandle
+ContinuousBatchingPipeline::IContinuousBatchingPipeline::add_request(
+    uint64_t request_id,
+    const std::string& prompt,
+    const std::vector<ov::Tensor>& images,
+    const std::vector<ov::Tensor>& videos,
+    GenerationConfig sampling_params) {
     OPENVINO_ASSERT(m_model_input_type == ModelInputType::EMBEDDINGS, "Model doesn't support embeddings.");
     ov::genai::VLMPerfMetrics metrics;
     ov::Tensor inputs;

src/cpp/src/continuous_batching/pipeline_base.hpp

@@ -55,6 +55,7 @@ class ContinuousBatchingPipeline::IContinuousBatchingPipeline {
     std::vector<size_t> m_history_image_ids;
     std::vector<ov::genai::EncodedVideo> m_history_videos;
     std::vector<size_t> m_history_video_ids;
+    std::vector<std::pair<std::size_t, std::size_t>> m_history_vision_count;  // pair<video count, image count>
     size_t m_image_id = 0;
     size_t m_video_id = 0;
 
@@ -144,7 +145,7 @@ class ContinuousBatchingPipeline::IContinuousBatchingPipeline {
 
     virtual std::vector<VLMDecodedResults> generate(const std::vector<std::string>& prompts,
                                                     const std::vector<std::vector<ov::Tensor>>& images,
-                                                    const std::vector<std::vector<ov::Tensor>>& video,
+                                                    const std::vector<std::vector<ov::Tensor>>& videos,
                                                     const std::vector<GenerationConfig>& sampling_params,
                                                     const StreamerVariant& streamer);
 

src/cpp/src/visual_language/clip.cpp

@@ -76,7 +76,12 @@ void bicubic_resize(const clip_image_u8 &img, clip_image_u8 &dst, int target_wid
 
     dst.nx = target_width;
     dst.ny = target_height;
-    dst.buf.resize(3 * target_width * target_height);
+    const int target_size = 3 * target_width * target_height;
+    dst.buf.resize(target_size);
+    if (img.nx == target_width && img.ny == target_height) {
+        std::memcpy(dst.buf.data(), img.buf.data(), target_size);
+        return;
+    }
 
     float Cc;
     float C[5];

src/cpp/src/visual_language/continuous_batching_adapter.hpp

@@ -53,14 +53,16 @@ class ov::genai::VLMPipeline::VLMContinuousBatchingAdapter : public ov::genai::V
     VLMDecodedResults generate(
         const std::string& prompt,
         const std::vector<ov::Tensor>& images,
-        const std::vector<ov::Tensor>& video,
+        const std::vector<ov::Tensor>& videos,
         GenerationConfig generation_config,
         const StreamerVariant& streamer
     ) override {
         auto start_time = std::chrono::steady_clock::now();
-        auto result = m_impl.generate({prompt}, {images}, {video}, {generation_config}, streamer)[0];
+        auto images_vec = images.size() == 0u ? std::vector<std::vector<ov::Tensor>>{} : std::vector<std::vector<ov::Tensor>>{images};
+        auto video_vec = videos.size() == 0u ? std::vector<std::vector<ov::Tensor>>{} : std::vector<std::vector<ov::Tensor>>{videos};
+        auto result = m_impl.generate({prompt}, images_vec, video_vec, {generation_config}, streamer)[0];
         auto stop_time = std::chrono::steady_clock::now();
-        
+
         VLMDecodedResults decoded;
         decoded.perf_metrics = result.perf_metrics;
         decoded.perf_metrics.load_time = get_load_time();

src/cpp/src/visual_language/gemma3/classes.cpp

@@ -85,7 +85,7 @@ std::vector<ov::genai::EncodedImage> InputsEmbedderGemma3::encode_images(const s
     return embeds;
 }
 
-NormlizedPrompt InputsEmbedderGemma3::normalize_prompt(const std::string& prompt, size_t base_id, const std::vector<EncodedImage>& images) const {
+NormalizedPrompt InputsEmbedderGemma3::normalize_prompt(const std::string& prompt, size_t base_id, const std::vector<EncodedImage>& images) const {
     std::string start_of_image = m_vlm_config.start_of_image;
     std::string image_token = m_vlm_config.image_soft_token;
     std::string end_of_image = m_vlm_config.end_of_image;

src/cpp/src/visual_language/gemma3/classes.hpp

@@ -43,7 +43,7 @@ class InputsEmbedderGemma3 : public InputsEmbedder::IInputsEmbedder {
 
     std::vector<ov::genai::EncodedImage> encode_images(const std::vector<ov::Tensor>& images) override;
 
-    NormlizedPrompt normalize_prompt(const std::string& prompt, size_t base_id, const std::vector<EncodedImage>& images) const override;
+    NormalizedPrompt normalize_prompt(const std::string& prompt, size_t base_id, const std::vector<EncodedImage>& images) const override;
 
     std::pair<ov::Tensor, std::optional<int64_t>> get_position_ids(const size_t inputs_embeds_size, const size_t history_size) override;