[wwb] Add custom processor for qwen3_vl (openvinotoolkit#3487)

## Description
New model should avoid using of preprocessing from optimum-intel.

<!-- Jira ticket number (e.g., 123). Delete if there's no ticket. -->
as part of the task
[CVS-175205](https://jira.devtools.intel.com/browse/CVS-175205)

## Checklist:
- [x] This PR follows [GenAI Contributing
guidelines](https://github.com/openvinotoolkit/openvino.genai?tab=contributing-ov-file#contributing).
<!-- Always follow them. If there are deviations, explain what and why.
-->
- [ ] Tests have been updated or added to cover the new code. <!--
Specify exactly which tests were added or updated. If the change isn't
maintenance related, update the tests at
https://github.com/openvinotoolkit/openvino.genai/tree/master/tests or
explain in the description why the tests don't need an update. -->
- [ ] This PR fully addresses the ticket. <!--- If not, explain clearly
what is covered and what is not. If follow-up pull requests are needed,
specify in the description. -->
- [ ] I have made corresponding changes to the documentation. <!-- Run
github.com/\<username>/openvino.genai/actions/workflows/deploy_gh_pages.yml
on your fork with your branch as a parameter to deploy a test version
with the updated content. Replace this comment with the link to the
built docs. If the documentation is updated in a separate PR, clearly
specify it. -->

Author: sbalandi

tools/who_what_benchmark/whowhatbench/visual_utils.py

@@ -0,0 +1,99 @@
+import numpy as np
+from transformers import (
+    AutoImageProcessor,
+    PretrainedConfig,
+    PreTrainedTokenizer,
+)
+from abc import ABC, abstractmethod
+from packaging.version import Version
+from typing import TYPE_CHECKING, Optional
+
+from transformers import __version__
+
+if TYPE_CHECKING:
+    from PIL.Image import Image
+    from transformers.image_utils import VideoInput
+
+
+TRANSFORMERS_VERSION = Version(__version__)
+
+
+def fix_phi3_v_eos_token_id(model_type, tokenizer):
+    """
+    phi3_v configs aren't consistent. Override the default
+    eos_token_id with the one from a tokenizer similar to
+    an example in
+    https://huggingface.co/microsoft/Phi-3.5-vision-instruct
+    """
+    if "phi3_v" == model_type:
+        return {"eos_token_id": tokenizer.eos_token_id}
+    else:
+        return dict()
+
+
+class VLMInputsPreprocessor(ABC):
+    def __init__(self, chat_mode: bool = False):
+        self.images = None
+        self.videos = []
+        self.chat_history = []
+        self.chat_mode = chat_mode
+
+    @abstractmethod
+    def preprocess_inputs(
+        self,
+        text: str,
+        image: Optional["Image"] = None,
+        processor: Optional[AutoImageProcessor] = None,
+        tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
+        video: Optional["VideoInput"] = None,
+        audio: Optional[np.ndarray] = None,
+    ):
+        return None
+
+    @abstractmethod
+    def update_chat_history_with_answer(self, answer):
+        pass
+
+
+class Qwen3VLInputsPreprocessor(VLMInputsPreprocessor):
+    def __init__(self, chat_mode: bool = False):
+        super().__init__(chat_mode)
+
+    def update_chat_history_with_answer(self, answer):
+        self.chat_history.append({"role": "assistant", "content": answer})
+
+    def preprocess_inputs(
+        self,
+        text: str,
+        image: Optional["Image"] = None,
+        processor: Optional[AutoImageProcessor] = None,
+        tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
+        video: Optional["VideoInput"] = None,
+        audio: Optional[np.ndarray] = None,
+    ):
+        if processor is None:
+            raise ValueError("Processor is required.")
+        if audio is not None:
+            raise ValueError("Audio input is not supported")
+        conversation = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text},
+                ],
+            }
+        ]
+        if image is not None:
+            conversation[0]["content"].insert(0, {"type": "image", "image": image})
+        if video is not None:
+            conversation[0]["content"].insert(0, {"type": "video", "video": video})
+
+        inputs = processor.apply_chat_template(
+            conversation, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt"
+        )
+        return inputs
+
+
+MODEL_TYPE_TO_CLS_MAPPING = {"qwen3_vl": Qwen3VLInputsPreprocessor}

tools/who_what_benchmark/whowhatbench/visualtext_evaluator.py

@@ -12,23 +12,11 @@
 from .registry import register_evaluator
 from .text_evaluator import TextEvaluator
 from .utils import get_ignore_parameters_flag, prepare_default_data_image, prepare_default_data_video
+from .visual_utils import fix_phi3_v_eos_token_id, MODEL_TYPE_TO_CLS_MAPPING
 
 DEF_VIDEO_FRAMES_AMOUNT = 10
 
 
-def fix_phi3_v_eos_token_id(model_type, tokenizer):
-    """
-    phi3_v configs aren't consistent. Override the default
-    eos_token_id with the one from a tokenizer similar to
-    an example in
-    https://huggingface.co/microsoft/Phi-3.5-vision-instruct
-    """
-    if 'phi3_v' == model_type:
-        return {"eos_token_id": tokenizer.eos_token_id}
-    else:
-        return dict()
-
-
 @register_evaluator("visual-text", "visual-video-text")
 class VisualTextEvaluator(TextEvaluator):
     def __init__(
@@ -128,12 +116,15 @@ def default_gen_answer(
             pruning_ratio,
             relevance_weight,
         ):
+            if model.config.model_type in MODEL_TYPE_TO_CLS_MAPPING and "transformers" in str(type(model)):
+                inputs_processor = MODEL_TYPE_TO_CLS_MAPPING[model.config.model_type]()
+                preprocess_inputs = inputs_processor.preprocess_inputs
+            else:
+                from optimum.intel.openvino.modeling_visual_language import (
+                    MODEL_TYPE_TO_CLS_MAPPING as MODEL_TYPE_TO_CLS_MAPPING_OPT,
+                )
 
-            from optimum.intel.openvino.modeling_visual_language import \
-                MODEL_TYPE_TO_CLS_MAPPING
-            preprocess_inputs = MODEL_TYPE_TO_CLS_MAPPING[
-                model.config.model_type
-            ].preprocess_inputs
+                preprocess_inputs = MODEL_TYPE_TO_CLS_MAPPING_OPT[model.config.model_type].preprocess_inputs
             inputs = preprocess_inputs(prompt, image, processor, tokenizer, config=model.config, video=video)
             tokens = model.generate(
                 **inputs,