Fix inference issue when only provides text for Qwen3-Omni

ZiniuLin · ZiniuLin · commit b6e489ee51cc · 2026-03-16T11:36:04.000+08:00
Fix inference issue when only provides text for Qwen3-Omni.

Signed-off-by: Ziniu Lin &lt;ziniu.lin@intel.com&gt;
diff --git a/samples/cpp/module_genai/config_yaml/Qwen3-Omni/config_prompt.yaml b/samples/cpp/module_genai/config_yaml/Qwen3-Omni/config_prompt.yaml
@@ -0,0 +1,71 @@
+global_context:
+  model_type: "qwen3_omni"
+
+pipeline_modules:
+  pipeline_params:
+    type: "ParameterModule"
+    outputs:
+      - name: "prompts"
+        type: "VecString"
+
+  prompt_encoder:
+    type: "TextEncoderModule"
+    device: "GPU"
+    inputs:
+      - name: "prompts"
+        type: "VecString"
+        source: "pipeline_params.prompts"
+    outputs:
+      - name: "input_ids"
+        type: "OVTensor"
+      - name: "mask"
+        type: "OVTensor"
+    params:
+      model_path: "./tests/module_genai/cpp/test_models/Qwen3-Omni-4B-Instruct-multilingual/"
+
+  vision_encoder:
+    type: "VisionEncoderModule"
+    device: "GPU"
+    inputs:
+      - name: "input_ids"
+        type: "OVTensor"
+        source: "prompt_encoder.input_ids"
+      - name: "attention_mask"
+        type: "OVTensor"
+        source: "prompt_encoder.mask"
+    outputs:
+      - name: "position_ids"
+        type: "OVTensor"
+      - name: "rope_delta"
+        type: "OVTensor"
+    params:
+      model_path: "./tests/module_genai/cpp/test_models/Qwen3-Omni-4B-Instruct-multilingual/qwen3_omni_vision_model.xml"
+      vision_start_token_id: 248053
+
+  llm:
+    type: "LLMInferenceSDPAModule"
+    device: "GPU"
+    inputs:
+      - name: "input_ids"
+        type: "OVTensor"
+        source: "prompt_encoder.input_ids"
+      - name: "position_ids"
+        type: "OVTensor"
+        source: "vision_encoder.position_ids"
+      - name: "rope_delta"
+        type: "OVTensor"
+        source: "vision_encoder.rope_delta"
+    outputs:
+      - name: "generated_text"
+        type: "String"
+    params:
+      model_path: "./tests/module_genai/cpp/test_models/Qwen3-Omni-4B-Instruct-multilingual/qwen3_omni_text_model.xml"
+      max_new_tokens: 512
+
+  pipeline_result:
+    type: "ResultModule"
+    description: "Collects final results and formats the output structure."
+    inputs:
+      - name: "generated_text"
+        type: "String"
+        source: "llm.generated_text"
diff --git a/src/cpp/src/module_genai/modules/md_llm_inference_sdpa.cpp b/src/cpp/src/module_genai/modules/md_llm_inference_sdpa.cpp
@@ -188,6 +188,7 @@ int64_t LLMInferenceSDPAModule::argmax_last(const ov::Tensor& logits) {
 
 bool LLMInferenceSDPAModule::initialize() {
     const auto& params = module_desc->params;
+    VLMModelType model_type = to_vlm_model_type(module_desc->model_type);
 
     // Resolve model directory
     std::filesystem::path models_path = get_optional_param("model_path");
@@ -224,7 +225,10 @@ bool LLMInferenceSDPAModule::initialize() {
 
     // Load model config
     try {
-        m_model_config = ov::genai::modeling::models::Qwen3_5Config::from_json_file(models_path);
+        if (model_type == VLMModelType::QWEN3_5)
+            m_model_config = ov::genai::modeling::models::Qwen3_5Config::from_json_file(models_path);
+        else if (model_type == VLMModelType::QWEN3_OMNI)
+            m_model_config = ov::genai::modeling::models::Qwen3OmniConfig::from_json_file(models_path);
     } catch (const std::exception& e) {
         GENAI_ERR("Failed to load Qwen3.5 config from " + models_path.string() + ": " + e.what());
         return false;
@@ -311,8 +315,11 @@ bool LLMInferenceSDPAModule::initialize() {
             if (eid >= 0) m_stop_ids.insert(eid);
         } catch (...) {}
     }
-    if (m_model_config.text.eos_token_id > 0) {
-        m_stop_ids.insert(m_model_config.text.eos_token_id);
+    if (model_type == VLMModelType::QWEN3_5) {
+        auto& cfg = std::get<ov::genai::modeling::models::Qwen3_5Config>(m_model_config);
+        if (cfg.text.eos_token_id > 0) {
+            m_stop_ids.insert(cfg.text.eos_token_id);
+        }
     }
     if (m_stop_ids.empty()) {
         GENAI_INFO("LLMInferenceSDPAModule: no stop token ids found — "
@@ -334,6 +341,7 @@ std::string LLMInferenceSDPAModule::run_text_decode(const ov::Tensor& input_ids,
                                                      const ov::Tensor& position_ids,
                                                      const ov::Tensor& rope_deltas) {
     using TIO = ov::genai::modeling::models::Qwen3_5TextIO;
+    auto model_config = std::get<modeling::models::Qwen3_5Config>(m_model_config);
 
     const size_t  batch      = input_ids.get_shape()[0];
     const int64_t prompt_len = static_cast<int64_t>(input_ids.get_shape()[1]);
@@ -352,7 +360,7 @@ std::string LLMInferenceSDPAModule::run_text_decode(const ov::Tensor& input_ids,
         // Feed zero visual inputs for text-only usage of VL IR
         text_req.set_tensor(TIO::kVisualEmbeds,
             make_zeros(ov::element::f32, {batch, static_cast<size_t>(prompt_len),
-                       static_cast<size_t>(m_model_config.text.hidden_size)}));
+                       static_cast<size_t>(model_config.text.hidden_size)}));
         text_req.set_tensor(TIO::kVisualPosMask,
             make_zeros(ov::element::boolean, {batch, static_cast<size_t>(prompt_len)}));
     }
@@ -373,7 +381,7 @@ std::string LLMInferenceSDPAModule::run_text_decode(const ov::Tensor& input_ids,
 
     ov::Tensor dec_vis, dec_vis_mask;
     if (m_text_uses_vl_ir) {
-        dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)});
+        dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(model_config.text.hidden_size)});
         dec_vis_mask = make_zeros(ov::element::boolean, {batch, 1});
     }
 
@@ -458,6 +466,7 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
                                                     const ov::Tensor& visual_pos_mask,
                                                     const std::optional<std::vector<ov::Tensor>>& deepstack_embeds) {
     using TIO = ov::genai::modeling::models::Qwen3_5TextIO;
+    auto model_config = std::get<modeling::models::Qwen3_5Config>(m_model_config);
 
     const size_t  batch      = input_ids.get_shape()[0];
     const int64_t prompt_len = static_cast<int64_t>(input_ids.get_shape()[1]);
@@ -480,7 +489,7 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
                 std::to_string(i);
             text_req.set_tensor(name, deepstack_embeds.value()[i]);
         }
-        ov::Tensor prefill_audio_features(ov::element::f32, {batch, input_ids.get_shape()[1], static_cast<size_t>(m_model_config.text.hidden_size)});
+        ov::Tensor prefill_audio_features(ov::element::f32, {batch, input_ids.get_shape()[1], static_cast<size_t>(model_config.text.hidden_size)});
         std::memset(prefill_audio_features.data(), 0, prefill_audio_features.get_byte_size());
         text_req.set_tensor("audio_features", prefill_audio_features);
         ov::Tensor prefill_audio_pos_mask(ov::element::boolean, {batch, input_ids.get_shape()[1]});
@@ -502,17 +511,17 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
     ov::Tensor step_mask = make_zeros(ov::element::i64, {batch, 1});
     for (size_t b = 0; b < batch; ++b) step_mask.data<int64_t>()[b] = 1;
 
-    ov::Tensor dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)});
+    ov::Tensor dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(model_config.text.hidden_size)});
     ov::Tensor dec_vis_mask = make_zeros(ov::element::boolean, {batch, 1});
     ov::Tensor decode_audio_features =
-        make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)});
+        make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(model_config.text.hidden_size)});
     ov::Tensor decode_audio_pos_mask = make_zeros(ov::element::boolean, {batch, 1});
     std::vector<ov::Tensor> decode_deepstack;
     if (deepstack_embeds.has_value()) {
         decode_deepstack.reserve(deepstack_embeds.value().size());
         for (size_t i = 0; i < deepstack_embeds.value().size(); ++i) {
             decode_deepstack.push_back(
-                make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)}));
+                make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(model_config.text.hidden_size)}));
         }
     }
 
@@ -605,6 +614,7 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
                                       const std::optional<ov::Tensor>& audio_embeds,
                                       const std::optional<ov::Tensor>& audio_pos_mask) {
     using TIO = ov::genai::modeling::models::Qwen3OmniTextIO;
+    auto model_config = std::get<modeling::models::Qwen3OmniConfig>(m_model_config);
 
     const size_t  batch      = input_ids.get_shape()[0];
     const int64_t prompt_len = static_cast<int64_t>(input_ids.get_shape()[1]);
@@ -621,6 +631,12 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
     if (visual_embeds.has_value() && visual_pos_mask.has_value()) {
         text_req.set_tensor(TIO::kVisualEmbeds,  visual_embeds.value());
         text_req.set_tensor(TIO::kVisualPosMask, visual_pos_mask.value());
+    } else{
+        text_req.set_tensor(TIO::kVisualEmbeds,
+            make_zeros(ov::element::f32, {batch, static_cast<size_t>(prompt_len),
+                       static_cast<size_t>(model_config.thinker.text.hidden_size)}));
+        text_req.set_tensor(TIO::kVisualPosMask,
+            make_zeros(ov::element::boolean, {batch, static_cast<size_t>(prompt_len)}));
     }
     if (deepstack_embeds.has_value()) {
         for (size_t i = 0; i < deepstack_embeds->size(); i++) {
@@ -629,12 +645,19 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
                 std::to_string(i);
             text_req.set_tensor(name, deepstack_embeds.value()[i]);
         }
+    } else {
+        for (size_t i = 0; i < model_config.thinker.vision.deepstack_visual_indexes.size(); i++) {
+            const std::string name =
+                std::string(ov::genai::modeling::models::Qwen3VLTextIO::kDeepstackEmbedsPrefix) + "." +
+                std::to_string(i);
+            text_req.set_tensor(name, make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(model_config.thinker.text.hidden_size)}));
+        }
     }
     if (audio_embeds.has_value() && audio_pos_mask.has_value()) {
         text_req.set_tensor(TIO::kAudioFeatures, audio_embeds.value());
         text_req.set_tensor(TIO::kAudioPosMask, audio_pos_mask.value());
     } else {
-        ov::Tensor prefill_audio_features(ov::element::f32, {batch, input_ids.get_shape()[1], static_cast<size_t>(m_model_config.text.hidden_size)});
+        ov::Tensor prefill_audio_features(ov::element::f32, {batch, input_ids.get_shape()[1], static_cast<size_t>(model_config.thinker.text.hidden_size)});
         std::memset(prefill_audio_features.data(), 0, prefill_audio_features.get_byte_size());
         text_req.set_tensor(TIO::kAudioFeatures, prefill_audio_features);
         ov::Tensor prefill_audio_pos_mask(ov::element::boolean, {batch, input_ids.get_shape()[1]});
@@ -656,18 +679,16 @@ std::string LLMInferenceSDPAModule::run_vl_decode(const ov::Tensor& input_ids,
     ov::Tensor step_mask = make_zeros(ov::element::i64, {batch, 1});
     for (size_t b = 0; b < batch; ++b) step_mask.data<int64_t>()[b] = 1;
 
-    ov::Tensor dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)});
+    ov::Tensor dec_vis      = make_zeros(ov::element::f32,     {batch, 1, static_cast<size_t>(model_config.thinker.text.hidden_size)});
     ov::Tensor dec_vis_mask = make_zeros(ov::element::boolean, {batch, 1});
     ov::Tensor decode_audio_features =
-        make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)});
+        make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(model_config.thinker.text.hidden_size)});
     ov::Tensor decode_audio_pos_mask = make_zeros(ov::element::boolean, {batch, 1});
     std::vector<ov::Tensor> decode_deepstack;
-    if (deepstack_embeds.has_value()) {
-        decode_deepstack.reserve(deepstack_embeds.value().size());
-        for (size_t i = 0; i < deepstack_embeds.value().size(); ++i) {
-            decode_deepstack.push_back(
-                make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(m_model_config.text.hidden_size)}));
-        }
+    decode_deepstack.reserve(model_config.thinker.vision.deepstack_visual_indexes.size());
+    for (size_t i = 0; i < model_config.thinker.vision.deepstack_visual_indexes.size(); ++i) {
+        decode_deepstack.push_back(
+            make_zeros(ov::element::f32, {batch, 1, static_cast<size_t>(model_config.thinker.text.hidden_size)}));
     }
 
     int64_t past_len     = prompt_len;
@@ -831,7 +852,7 @@ void LLMInferenceSDPAModule::run() {
                         this->inputs.find("position_ids") != this->inputs.end() &&
                         this->inputs.find("rope_delta") != this->inputs.end());
 
-    ov::genai::modeling::models::Qwen3_5InputPlanner planner(m_model_config);
+    ov::genai::modeling::models::Qwen3_5InputPlanner planner(std::get<modeling::models::Qwen3_5Config>(m_model_config));
 
     if (is_vl) {
         // ---- VL mode ----
diff --git a/src/cpp/src/module_genai/modules/md_llm_inference_sdpa.hpp b/src/cpp/src/module_genai/modules/md_llm_inference_sdpa.hpp
@@ -13,6 +13,7 @@
 #include "modeling/models/qwen3_5/modeling_qwen3_5_text.hpp"
 #include "modeling/models/qwen3_5/processing_qwen3_5.hpp"
 #include "modeling/weights/quantization_config.hpp"
+#include "modeling/models/qwen3_omni/processing_qwen3_omni.hpp"
 
 namespace ov {
 namespace genai {
@@ -82,7 +83,7 @@ class LLMInferenceSDPAModule : public IBaseModule {
     std::set<int64_t> m_stop_ids;
 
     // Model config
-    ov::genai::modeling::models::Qwen3_5Config m_model_config;
+    std::variant<ov::genai::modeling::models::Qwen3_5Config, modeling::models::Qwen3OmniConfig> m_model_config;
 
     // Tokenizer (for text mode and decoding)
     std::unique_ptr<ov::genai::Tokenizer> m_tokenizer;
