mean, std, change to const.

Signed-off-by: xipingya <xiping.yan@intel.com>

Author: xipingyan

src/cpp/src/visual_language/inputs_embedder.cpp

@@ -166,21 +166,13 @@ std::vector<ov::Tensor> InputsEmbedder::IInputsEmbedder::to_single_image_tensors
 std::vector<ov::genai::EncodedImage> InputsEmbedder::IInputsEmbedder::encode_images(const std::vector<ov::Tensor>& images, const bool& is_video) {
     std::vector<ov::Tensor> single_images = to_single_image_tensors(images);
     if (is_video) {
-        auto t1 = std::chrono::high_resolution_clock::now();
-        auto res = m_vision_encoder->encode_video(single_images);
-        auto t2 = std::chrono::high_resolution_clock::now();
-        std::cout << "encode video time: " << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
-                  << std::endl;
+        return m_vision_encoder->encode_video(single_images);
     }
 
-    auto t1 = std::chrono::high_resolution_clock::now();
     std::vector<EncodedImage> embeds;
     for (const ov::Tensor& image : single_images) {
         embeds.emplace_back(m_vision_encoder->encode(image));
     }
-    auto t2 = std::chrono::high_resolution_clock::now();
-    std::cout << "image video time: " << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
-              << std::endl;
     return embeds;
 }
 

src/cpp/src/visual_language/qwen2vl/classes.cpp

@@ -31,10 +31,9 @@ const std::string NATIVE_TAG = "<|vision_start|><|image_pad|><|vision_end|>";
 std::pair<std::shared_ptr<ov::Model>, std::shared_ptr<ov::op::v0::Result>> patch_preprocess_branch_image(
     std::shared_ptr<ov::op::v0::Parameter> raw_images_1,
     std::shared_ptr<ov::op::v0::Parameter> resize_target_shape,
-    std::shared_ptr<ov::op::v0::Parameter> image_mean,
-    std::shared_ptr<ov::op::v0::Parameter> image_scale,
+    std::shared_ptr<ov::op::v0::Constant> image_mean,
+    std::shared_ptr<ov::op::v0::Constant> image_scale,
     std::shared_ptr<ov::op::v0::Parameter> broadcast_shape) {
-    std::cout << "patch_preprocess_branch_image" << std::endl;
     auto raw_images_f32_1 = std::make_shared<ov::op::v0::Convert>(raw_images_1, ov::element::f32);
     auto img_trans_1 = std::make_shared<ov::op::v1::Transpose>(
         raw_images_f32_1,
@@ -55,11 +54,7 @@ std::pair<std::shared_ptr<ov::Model>, std::shared_ptr<ov::op::v0::Result>> patch
     auto temporal_images = std::make_shared<ov::op::v3::Broadcast>(resized_images_s_1, broadcast_shape);
     auto results = std::make_shared<ov::op::v0::Result>(temporal_images);
     return {std::make_shared<ov::Model>(results,
-                                        ov::ParameterVector{raw_images_1,
-                                                            resize_target_shape,
-                                                            image_mean,
-                                                            image_scale,
-                                                            broadcast_shape},
+                                        ov::ParameterVector{raw_images_1, resize_target_shape, broadcast_shape},
                                         "then_body"),
             results};
 }
@@ -69,21 +64,16 @@ std::pair<std::shared_ptr<ov::Model>, std::shared_ptr<ov::op::v0::Result>> patch
     std::shared_ptr<ov::op::v0::Parameter> raw_images_1,
     std::shared_ptr<ov::op::v0::Parameter> raw_images_2,
     std::shared_ptr<ov::op::v0::Parameter> resize_target_shape,
-    std::shared_ptr<ov::op::v0::Parameter> image_mean,
-    std::shared_ptr<ov::op::v0::Parameter> image_scale) {
-    std::cout << "patch_preprocess_branch_video" << std::endl;
+    std::shared_ptr<ov::op::v0::Constant> image_mean,
+    std::shared_ptr<ov::op::v0::Constant> image_scale) {
     auto raw_images_f32_1 = std::make_shared<ov::op::v0::Convert>(raw_images_1, ov::element::f32);
     auto raw_images_f32_2 = std::make_shared<ov::op::v0::Convert>(raw_images_2, ov::element::f32);
     auto img_trans_1 = std::make_shared<ov::op::v1::Transpose>(
         raw_images_f32_1,
         std::make_shared<ov::op::v0::Constant>(ov::element::i32, Shape{4}, std::vector<int32_t>{0, 3, 1, 2}));
-    img_trans_1->set_friendly_name("img_trans_1");
-    img_trans_1->output(0).get_tensor().set_names({"img_trans_1"});
     auto img_trans_2 = std::make_shared<ov::op::v1::Transpose>(
         raw_images_f32_2,
         std::make_shared<ov::op::v0::Constant>(ov::element::i32, Shape{4}, std::vector<int32_t>{0, 3, 1, 2}));
-    img_trans_2->set_friendly_name("img_trans_2");
-    img_trans_2->output(0).get_tensor().set_names({"img_trans_2"});
 
     ov::op::v0::Interpolate::Attributes attrs = {};
     attrs.axes = {2, 3};
@@ -109,24 +99,28 @@ std::pair<std::shared_ptr<ov::Model>, std::shared_ptr<ov::op::v0::Result>> patch
 
     auto result_temperal_images = std::make_shared<ov::op::v0::Result>(temporal_images);
     auto result_tmp = std::make_shared<ov::op::v0::Result>(same_image);
-    return {std::make_shared<ov::Model>(
-                ov::ResultVector{result_temperal_images, result_tmp},
-                ov::ParameterVector{same_image, raw_images_1, raw_images_2, resize_target_shape, image_mean, image_scale},
-                "else_body"),
-            result_temperal_images};
+    return {
+        std::make_shared<ov::Model>(ov::ResultVector{result_temperal_images, result_tmp},
+                                    ov::ParameterVector{same_image, raw_images_1, raw_images_2, resize_target_shape},
+                                    "else_body"),
+        result_temperal_images};
 }
 
-std::shared_ptr<ov::Model> patch_preprocess_into_model(std::shared_ptr<ov::Model> model_org) {
-    std::cout << "patch_preprocess_into_model" << std::endl;
+std::shared_ptr<ov::Model> patch_preprocess_into_model(std::shared_ptr<ov::Model> model_org, const ProcessorConfig& config) {    
+    std::vector<float> a_image_mean(config.image_mean.begin(), config.image_mean.end());
+    std::vector<float> a_image_std(config.image_std.begin(), config.image_std.end());
+    for (auto& v : a_image_mean) v *= 255.0f;
+    for (auto& v : a_image_std) v = 1.0f / (v * 255.0f);
+    auto image_mean = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{1, config.image_mean.size(), 1, 1}, a_image_mean.data());
+    auto image_std = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{1, config.image_mean.size(), 1, 1}, a_image_std.data());
+
     auto same_image = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::Shape{1});
     same_image->set_friendly_name("same_image");
     same_image->output(0).get_tensor().set_names({"same_image"});
 
     auto raw_images_1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
     auto raw_images_2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
     auto resize_target_shape = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{2});
-    auto image_mean = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
-    auto image_scale = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
 
     auto broadcast_shape = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{4});
     auto temp_shape8d = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{8});
@@ -141,39 +135,28 @@ std::shared_ptr<ov::Model> patch_preprocess_into_model(std::shared_ptr<ov::Model
     resize_target_shape->set_friendly_name("resize_target_shape");
     resize_target_shape->output(0).get_tensor().set_names({"resize_target_shape"});
 
-    image_mean->set_friendly_name("image_mean");
-    image_mean->output(0).get_tensor().set_names({"image_mean"});
-
-    image_scale->set_friendly_name("image_scale");
-    image_scale->output(0).get_tensor().set_names({"image_scale"});
-
     broadcast_shape->set_friendly_name("broadcast_shape");
     broadcast_shape->output(0).get_tensor().set_names({"broadcast_shape"});
 
     auto then_raw_images_1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
-    auto then_raw_images_2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
     auto then_resize_target_shape = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{2});
-    auto then_image_mean = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
-    auto then_image_scale = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
     auto then_broadcast_shape = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{4});
     auto model_then = patch_preprocess_branch_image(then_raw_images_1,
                                                     then_resize_target_shape,
-                                                    then_image_mean,
-                                                    then_image_scale,
+                                                    image_mean,
+                                                    image_std,
                                                     then_broadcast_shape);
 
     auto else_same_image = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::Shape{1});
     auto else_raw_images_1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
     auto else_raw_images_2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u8, ov::PartialShape{-1, -1, -1, -1});
     auto else_resize_target_shape = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::PartialShape{2});
-    auto else_image_mean = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
-    auto else_image_scale = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{1, -1, 1, 1});
     auto model_else = patch_preprocess_branch_video(else_same_image,
                                                     else_raw_images_1,
                                                     else_raw_images_2,
                                                     else_resize_target_shape,
-                                                    else_image_mean,
-                                                    else_image_scale);
+                                                    image_mean,
+                                                    image_std);
 
     auto if_op = std::make_shared<ov::op::v8::If>();
     if_op->set_then_body(model_then.first);
@@ -183,13 +166,9 @@ std::shared_ptr<ov::Model> patch_preprocess_into_model(std::shared_ptr<ov::Model
     if_op->set_input(raw_images_1->output(0), nullptr, else_raw_images_1);
     if_op->set_input(raw_images_2->output(0), nullptr, else_raw_images_2);
     if_op->set_input(resize_target_shape->output(0), nullptr, else_resize_target_shape);
-    if_op->set_input(image_mean->output(0), nullptr, else_image_mean);
-    if_op->set_input(image_scale->output(0), nullptr, else_image_scale);
 
     if_op->set_input(raw_images_1->output(0), then_raw_images_1, nullptr);
     if_op->set_input(resize_target_shape->output(0), then_resize_target_shape, nullptr);
-    if_op->set_input(image_mean->output(0), then_image_mean, nullptr);
-    if_op->set_input(image_scale->output(0), then_image_scale, nullptr);
     if_op->set_input(broadcast_shape->output(0), then_broadcast_shape, nullptr);
 
     auto temporal_images = if_op->set_output(model_then.second, model_else.second);
@@ -223,8 +202,6 @@ std::shared_ptr<ov::Model> patch_preprocess_into_model(std::shared_ptr<ov::Model
                                                            raw_images_1,
                                                            raw_images_2,
                                                            resize_target_shape,
-                                                           image_mean,
-                                                           image_scale,
                                                            broadcast_shape,
                                                            temp_shape8d,
                                                            temp_shape4d,
@@ -539,8 +516,9 @@ std::unique_ptr<CircularBufferQueue<ov::InferRequest>> VisionEncoderQwen2VL::cre
 }
 
 VisionEncoderQwen2VL::VisionEncoderQwen2VL(const std::filesystem::path& model_dir, const std::string& device, const ov::AnyMap properties) : VisionEncoder(model_dir, device, properties) {
+    ProcessorConfig config = utils::from_any_map({}, m_processor_config);
     auto model_org = utils::singleton_core().read_model(model_dir / "openvino_vision_embeddings_model.xml");
-    auto model = patch_preprocess_into_model(model_org);
+    auto model = patch_preprocess_into_model(model_org, config);
     auto compiled_model = utils::singleton_core().compile_model(model, device, properties);
     m_ireq_queue_vision_encoder = create_ireq(compiled_model);
 }
@@ -552,8 +530,9 @@ VisionEncoderQwen2VL::VisionEncoderQwen2VL(const ModelsMap& models_map,
     const auto& vision_encoder_model = utils::get_model_weights_pair(models_map, "vision_embeddings").first;
     const auto& vision_encoder_weights = utils::get_model_weights_pair(models_map, "vision_embeddings").second;
 
+    ProcessorConfig config = utils::from_any_map({}, m_processor_config);
     auto model_org = utils::singleton_core().read_model(vision_encoder_model, vision_encoder_weights);
-    auto model = patch_preprocess_into_model(model_org);
+    auto model = patch_preprocess_into_model(model_org, config);
 
     auto compiled_model = utils::singleton_core().compile_model(model, device, device_config);
     m_ireq_queue_vision_encoder = create_ireq(compiled_model);
@@ -644,23 +623,13 @@ std::vector<EncodedImage> VisionEncoderQwen2VL::encode_video(const std::vector<o
         ov::Tensor temp_shape8d(ov::element::i64, ov::Shape{8}, a_temp_shape8d);
         ov::Tensor temp_shape4d(ov::element::i64, ov::Shape{4}, a_temp_shape4d);
         ov::Tensor last_shape(ov::element::i64, ov::Shape{2}, last_output_shape);
-        
-        std::vector<float> a_image_mean(config.image_mean.begin(), config.image_mean.end());
-        std::vector<float> a_image_scale(config.image_std.begin(), config.image_std.end());
-        for(auto& v : a_image_mean) v *= 255.0f;
-        for(auto& v : a_image_scale) v = 1.0f / (v*255.0f);
-    
-        ov::Tensor image_mean(ov::element::f32, ov::Shape{1,a_image_mean.size(),1,1}, a_image_mean.data());
-        ov::Tensor image_scale(ov::element::f32, ov::Shape{1,a_image_scale.size(),1,1}, a_image_scale.data());
-    
+
         ov::Tensor same_image(ov::element::f32, ov::Shape{1}, std::vector<float>{0}.data());
 
         encoder.set_tensor("same_image", same_image);
         encoder.set_tensor("raw_images_1", raw_image_1);
         encoder.set_tensor("raw_images_2", raw_image_2);
         encoder.set_tensor("resize_target_shape", target_shape);
-        encoder.set_tensor("image_mean", image_mean);
-        encoder.set_tensor("image_scale", image_scale);
         encoder.set_tensor("broadcast_shape", broadcast_shape);
         encoder.set_tensor("temp_shape8d", temp_shape8d);
         encoder.set_tensor("temp_shape4d", temp_shape4d);
@@ -722,22 +691,12 @@ EncodedImage VisionEncoderQwen2VL::encode(const ov::Tensor& image, const ov::Any
     ov::Tensor temp_shape8d(ov::element::i64, ov::Shape{8}, a_temp_shape8d);
     ov::Tensor temp_shape4d(ov::element::i64, ov::Shape{4}, a_temp_shape4d);
     ov::Tensor last_shape(ov::element::i64, ov::Shape{2}, last_output_shape);
-    
-    std::vector<float> a_image_mean(config.image_mean.begin(), config.image_mean.end());
-    std::vector<float> a_image_scale(config.image_std.begin(), config.image_std.end());
-    for(auto& v : a_image_mean) v *= 255.0f;
-    for(auto& v : a_image_scale) v = 1.0f / (v*255.0f);
-
-    ov::Tensor image_mean(ov::element::f32, ov::Shape{1,a_image_mean.size(),1,1}, a_image_mean.data());
-    ov::Tensor image_scale(ov::element::f32, ov::Shape{1,a_image_scale.size(),1,1}, a_image_scale.data());
-
     ov::Tensor same_image(ov::element::f32, ov::Shape{1}, std::vector<float>{0}.data());
+
     encoder.set_tensor("same_image", same_image);
     encoder.set_tensor("raw_images_1", raw_images);
     encoder.set_tensor("raw_images_2", raw_images);
     encoder.set_tensor("resize_target_shape", target_shape);
-    encoder.set_tensor("image_mean", image_mean);
-    encoder.set_tensor("image_scale", image_scale);
     encoder.set_tensor("broadcast_shape", broadcast_shape);
     encoder.set_tensor("temp_shape8d", temp_shape8d);
     encoder.set_tensor("temp_shape4d", temp_shape4d);