pipeline.hpp

// Copyright (C) 2023-2025 Intel Corporation
// SPDX-License-Identifier: Apache-2.0

#pragma once

#include <string>
#include <vector>
#include <filesystem>

#include "openvino/genai/llm_pipeline.hpp"
#include "openvino/genai/streamer_base.hpp"
#include "openvino/genai/tokenizer.hpp"
#include "openvino/genai/visual_language/perf_metrics.hpp"

namespace ov::genai {

class OPENVINO_GENAI_EXPORTS VLMDecodedResults : public DecodedResults{
public:
    VLMPerfMetrics perf_metrics;
};

/// @brief A Visual language modeling pipeline class used to generate a
/// response or run a chat given a prompt and an image.
class OPENVINO_GENAI_EXPORTS VLMPipeline {
public:
    /// @brief Construct a pipeline from a folder containing tokenizer
    /// and model IRs.
    /// @param models_path A folder to read tokenizer and model IRs.
    /// @param device Inference device. A tokenizer is always compiled
    /// for CPU.
    /// @param properties A config to pass to ov::Core::compile_model().
    VLMPipeline(
        const std::filesystem::path& models_path,
        const std::string& device,
        const ov::AnyMap& properties = {}
    );

    /// @brief Construct a pipeline from a map of models and their weights.
    /// @param models_map A map where key is model name (e.g. "vision_embeddings", "text_embeddings", "language", "resampler")
    /// and value is a pair of model IR as string and weights as tensor.
    /// @param tokenizer A tokenizer.
    /// @param config_dir_path A path to directory containing config.json.
    /// @param device Inference device. A tokenizer is always compiled
    /// for CPU.
    /// @param properties A config to pass to ov::Core::compile_model().
    /// @param generation_config Optional generation configuration for the pipeline.
    VLMPipeline(
        const ModelsMap& models_map,
        const Tokenizer& tokenizer,
        const std::filesystem::path& config_dir_path,
        const std::string& device,
        const ov::AnyMap& properties = {},
        const ov::genai::GenerationConfig& generation_config = {}
    );

    /// @brief Construct a pipeline from a folder containing tokenizer
    /// and model IRs. Accepts arbitrary list of optional properties.
    /// @param models_path A folder to read tokenizer and model IRs.
    /// @param device Inference device. A tokenizer is always compiled
    /// for CPU.
    /// @param properties A config to pass to ov::Core::compile_model().
    template <typename... Properties, typename std::enable_if<ov::util::StringAny<Properties...>::value, bool>::type = true>
    VLMPipeline(
        const std::filesystem::path& models_path,
        const std::string& device,
        Properties&&... properties)
        : VLMPipeline(models_path, device, ov::AnyMap{std::forward<Properties>(properties)...}) { }

    /// @brief Construct a pipeline from a map of models and their weights.
    /// @param models_map A map where key is model name (e.g. "vision_embeddings", "text_embeddings", "language", "resampler")
    /// and value is a pair of model IR as string and weights as tensor.
    /// @param tokenizer A tokenizer.
    /// @param config_dir_path A path to directory containing config.json.
    /// @param device Inference device. A tokenizer is always compiled
    /// for CPU.
    /// @param properties A config to pass to ov::Core::compile_model().
    template <typename... Properties, typename std::enable_if<ov::util::StringAny<Properties...>::value, bool>::type = true>
    VLMPipeline(
        const ModelsMap& models_map,
        const Tokenizer& tokenizer,
        const std::filesystem::path& config_dir_path,
        const std::string& device,
        Properties&&... properties)
        : VLMPipeline(models_map, tokenizer, config_dir_path, device, ov::AnyMap{std::forward<Properties>(properties)...}) { }

    /// @brief Default destructor.
    ~VLMPipeline();

    /// @brief Generate a response given a prompt and any number of
    /// uint8 RGB images with [NHWC] or [HWC] layout.
    /// @param prompt A prompt to respond to.
    /// @param images Images to be prepended to a prompt.
    /// @param generation_config A config to follow for text generation.
    /// @param streamer A streamer to acquire intermediate result.
    /// @return A string generated by a model.
    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
    VLMDecodedResults generate(
        const std::string& prompt,
        const std::vector<ov::Tensor>& images,
        const GenerationConfig& generation_config,
        const StreamerVariant& streamer
    );

    /// @brief Generate a response given a prompt and uint8 RGB image with [NHWC] or [HWC] layout.
    /// @param prompt A prompt to respond to.
    /// The prompt can contain <ov_genai_image_i> with i replaced with
    /// an actual zero based index to refer to an image. Reference to
    /// images used in previous prompts isn't implemented.
    /// A model's native image tag can be used instead of
    /// <ov_genai_image_i>. These tags are:
    /// InternVL2: <image>\n
    /// llava-1.5-7b-hf: <image>
    /// LLaVA-NeXT: <image>
    /// LLaVa-NeXT-Video: <image>
    /// nanoLLaVA: <image>\n
    /// nanoLLaVA-1.5: <image>\n
    /// MiniCPM-o-2_6: <image>./</image>\n
    /// MiniCPM-V-2_6: <image>./</image>\n
    /// Phi-3-vision: <|image_i|>\n - the index starts with one
    /// Phi-4-multimodal-instruct: <|image_i|>\n - the index starts with one
    /// Qwen2-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// Qwen2.5-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// gemma-3-4b-it: <start_of_image>
    /// Model's native video tag can be used to refer to a video:
    /// LLaVa-NeXT-Video: <video>
    /// If the prompt doesn't contain image or video tags, but images or videos are
    /// provided, the tags are prepended to the prompt.
    /// @param images Image to be prepended to a prompt.
    /// @param videos Multiple videos, each providing multiple frames, to be prepended to a prompt.
    /// @param generation_config A config to follow for text generation.
    /// @param streamer A streamer to acquire intermediate result.
    /// @return A string generated by a model.
    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
    VLMDecodedResults generate(
        const std::string& prompt,
        const std::vector<ov::Tensor>& images,
        const std::vector<ov::Tensor>& videos,
        const GenerationConfig& generation_config,
        const StreamerVariant& streamer
    );

    /// @brief Generate a response given a prompt and uint8 RGB image with [NHWC] or [HWC] layout.
    /// @param prompt A prompt to respond to.
    /// The prompt can contain <ov_genai_image_i> with i replaced with
    /// an actual zero based index to refer to an image. Reference to
    /// images used in previous prompts isn't implemented.
    /// A model's native image tag can be used instead of
    /// <ov_genai_image_i>. These tags are:
    /// InternVL2: <image>\n
    /// llava-1.5-7b-hf: <image>
    /// LLaVA-NeXT: <image>
    /// LLaVa-NeXT-Video: <image>
    /// nanoLLaVA: <image>\n
    /// nanoLLaVA-1.5: <image>\n
    /// MiniCPM-o-2_6: <image>./</image>\n
    /// MiniCPM-V-2_6: <image>./</image>\n
    /// Phi-3-vision: <|image_i|>\n - the index starts with one
    /// Phi-4-multimodal-instruct: <|image_i|>\n - the index starts with one
    /// Qwen2-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// Qwen2.5-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// gemma-3-4b-it: <start_of_image>
    /// If the prompt doesn't contain image tags, but images are
    /// provided, the tags are prepended to the prompt.
    /// @param images Image to be prepended to a prompt.
    /// @param generation_config A config to follow for text generation.
    /// @param streamer A streamer to acquire intermediate result.
    /// @return A string generated by a model.
    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
    VLMDecodedResults generate(
        const std::string& prompt,
        const ov::Tensor& images,
        const GenerationConfig& generation_config,
        const StreamerVariant& streamer
    );

    /// @brief Generate a response given a prompt and config.
    /// @param prompt A prompt to respond to.
    /// The prompt can contain <ov_genai_image_i> with i replaced with
    /// an actual zero based index to refer to an image. Reference to
    /// images used in previous prompts isn't implemented.
    /// A model's native image tag can be used instead of
    /// <ov_genai_image_i>. These tags are:
    /// InternVL2: <image>\n
    /// llava-1.5-7b-hf: <image>
    /// LLaVA-NeXT: <image>
    /// LLaVa-NeXT-Video: <image>
    /// nanoLLaVA: <image>\n
    /// nanoLLaVA-1.5: <image>\n
    /// MiniCPM-o-2_6: <image>./</image>\n
    /// MiniCPM-V-2_6: <image>./</image>\n
    /// Phi-3-vision: <|image_i|>\n - the index starts with one
    /// Phi-4-multimodal-instruct: <|image_i|>\n - the index starts with one
    /// Qwen2-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// Qwen2.5-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// gemma-3-4b-it: <start_of_image>
    /// Model's native video tag can be used to refer to a video:
    /// LLaVa-NeXT-Video: <video>
    /// If the prompt doesn't contain image or video tags, but images or videos are
    /// provided, the tags are prepended to the prompt.
    /// @param config_map A config may contain GenerationConfig, values
    /// for its members, StreamerVariant a single image or multiple
    /// images.
    /// @return A string generated by a model.
    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
    VLMDecodedResults generate(
        const std::string& prompt,
        const ov::AnyMap& config_map
    );

    /// @brief Generate a response given a prompt and arbitrary number
    /// of ov::Property instances.
    /// Example:
    /// generate("text", image(rgb), do_sample(true));
    /// @param prompt A prompt to respond to.
    /// The prompt can contain <ov_genai_image_i> with i replaced with
    /// an actual zero based index to refer to an image. Reference to
    /// images used in previous prompts isn't implemented.
    /// A model's native image tag can be used instead of
    /// <ov_genai_image_i>. These tags are:
    /// InternVL2: <image>\n
    /// llava-1.5-7b-hf: <image>
    /// LLaVA-NeXT: <image>
    /// LLaVa-NeXT-Video: <image>
    /// nanoLLaVA: <image>\n
    /// nanoLLaVA-1.5: <image>\n
    /// MiniCPM-o-2_6: <image>./</image>\n
    /// MiniCPM-V-2_6: <image>./</image>\n
    /// Phi-3-vision: <|image_i|>\n - the index starts with one
    /// Phi-4-multimodal-instruct: <|image_i|>\n - the index starts with one
    /// Qwen2-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// Qwen2.5-VL: <|vision_start|><|image_pad|><|vision_end|>
    /// gemma-3-4b-it: <start_of_image>
    /// If the prompt doesn't contain image tags, but images are
    /// provided, the tags are prepended to the prompt.
    /// @param ...properties ov::Property instances to be combined into
    /// ov::AnyMap.
    /// @return A string generated by a model.
    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
    template <typename... Properties>
    util::EnableIfAllStringAny<VLMDecodedResults, Properties...> generate(
        const std::string& prompt,
        Properties&&... properties
    ) {
        return generate(
            prompt, AnyMap{std::forward<Properties>(properties)...}
        );
    }

    /// @brief Activate chat mode. Chat preserves previous history.
    /// Calling start_chat() again or finish_chat() drops the memorized history.
    /// @param system_message Some chat_templates contain system role
    /// in addition to user and assistant roles. Set a message for that
    /// role.
    void start_chat(const std::string& system_message="");

    /// @brief Deactivate chat mode.
    void finish_chat();

    /// @brief Set a custom chat template. Can be used to deactivate
    /// chat_template application for chat mode if called with
    /// "{% for message in messages %}{{ message['content'] }}{% endfor %}"
    /// or workaround unsupported chat_template entries in a default
    /// model chat_template.
    /// @param new_template A new template to override with.
    void set_chat_template(const std::string& new_template);

    /// @brief Get a Tokenizer used to tokenize input and detokenize
    /// output.
    ov::genai::Tokenizer get_tokenizer() const;

    /// @brief Extract GenerationConfig used to get default values.
    /// @return Default values used.
    GenerationConfig get_generation_config() const;

    /// @brief Override default values for GenerationConfig
    /// @param new_config A config to override default values with.
    void set_generation_config(const GenerationConfig& new_config);

private:
    class VLMPipelineBase;
    class VLMPipelineImpl;
    class VLMContinuousBatchingAdapter;
    std::unique_ptr<VLMPipelineBase> m_pimpl;
};

/*
 * utils that allow to use generate() in the following way:
 * pipe.generate(prompt, ov::genai::image(image_tensor)).
 * pipe.generate(prompt, ov::genai::images(image_tensors)).
 * pipe.generate(prompt, ov::genai::videos(videos_tensors)).
*/
static constexpr ov::Property<ov::Tensor> image{"image"};
static constexpr ov::Property<std::vector<ov::Tensor>> images{"images"};
static constexpr ov::Property<std::vector<ov::Tensor>> videos{"videos"};
}