kcz/add_mp4_disabling_into_frames

Author: krzyczar

tools/llm_bench/benchmark.py

@@ -219,6 +219,8 @@ def get_argprser():
                         help="Path to .bin or .pt file with speaker embeddings for text to speech scenarios")
     parser.add_argument("--vocoder_path", type=str, default=None,
                         help="Path to vocoder  for text to speech scenarios")
+    parser.add_argument("--video_frames", type=int, default=None,
+                        help="number of video frames to process")
     return parser.parse_args()
 
 

tools/llm_bench/llm_bench_utils/prompt_utils.py

@@ -2,10 +2,13 @@
 # Copyright (C) 2023-2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+
+import os
+import cv2
+from PIL import Image
 from .model_utils import get_param_from_file
 from .parse_json_data import parse_text_json_data
 
-
 def get_text_prompt(args):
     text_list = []
     output_data_list, is_json_data = get_param_from_file(args, 'prompt')
@@ -17,3 +20,31 @@ def get_text_prompt(args):
     else:
         text_list.append(output_data_list[0])
     return text_list
+
+
+def split_video_into_frames(video_path, decym_frames=None):
+    supported_files = set([".mp4"])
+    assert os.path.exists(video_path), "no input video file"
+    assert video_path.suffix.lower() in supported_files, "no supported video file"
+    cap = cv2.VideoCapture(video_path)
+
+    output_frames = []
+    while True:
+        ret, frame = cap.read()
+        if not ret: break
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        pil_image = Image.fromarray(frame_rgb)
+        output_frames.append(pil_image)
+    if decym_frames is None:
+        return output_frames
+
+    # decimation procedure:
+    # decim_fames is required frame number
+    #
+    decym_frames = int(decym_frames)
+    if len(output_frames) <= decym_frames:
+        return output_frames
+    decym_factor = len(output_frames) / decym_frames
+    if decym_factor >= 2:
+        return list(output_frames[::decym_factor])
+    return output_frames

tools/llm_bench/requirements.txt

@@ -8,7 +8,7 @@ pillow
 torch
 transformers[sentencepiece]>=4.40.0
 diffusers>=0.22.0
-#optimum is in dependency list of optimum-intel 
+#optimum is in dependency list of optimum-intel
 optimum-intel[nncf]>=1.25.0
 packaging
 psutil
@@ -20,3 +20,4 @@ jinja2>=3.1.0
 scipy
 gguf_parser
 gguf>=0.10
+opencv-python

tools/llm_bench/task/visual_language_generation.py

@@ -17,9 +17,9 @@
 import llm_bench_utils.output_file
 import llm_bench_utils.gen_output_data as gen_output_data
 import llm_bench_utils.parse_json_data as parse_json_data
+import llm_bench_utils.prompt_utils as pu
 from pathlib import Path
 
-
 FW_UTILS = {'pt': llm_bench_utils.pt_utils, 'ov': llm_bench_utils.ov_utils}
 
 DEFAULT_OUTPUT_TOKEN_SIZE = 512
@@ -37,13 +37,20 @@ def run_visual_language_generation_optimum(
     prompts = []
     inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
     for input_data in inputs:
-        if input_data.get("media", None):
+        if input_data.get("video", None):
+            entry = Path(input_data["video"])
+            required_frames = args.get('video_frames')
+            ordered_frames = pu.split_video_into_frames(entry, required_frames)
+            images.extend(ordered_frames)
+
+        elif input_data.get("media", None):
             entry = Path(input_data["media"])
             if entry.is_dir():
                 for file in sorted(entry.iterdir()):
                     images.append(load_image(str(file)))
-            else:
-                images.append(load_image(input_data["media"]))
+            else: images.append(load_image(input_data["media"]))
+
+
         prompts.append(input_data["prompt"])
     prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
     log.info(f'{prefix}[P{prompt_index}] Input image nums:{len(images)}')
@@ -365,14 +372,13 @@ def run_visual_language_generation_benchmark(model_path, framework, device, args
 
 def get_image_text_prompt(args):
     vlm_file_list = []
-    output_data_list, is_json_data = model_utils.get_param_from_file(args, ['media', "prompt"])
+    output_data_list, is_json_data = model_utils.get_param_from_file(args, ["media", "prompt"])
     if is_json_data:
         vlm_param_list = parse_json_data.parse_vlm_json_data(output_data_list)
         if len(vlm_param_list) > 0:
             for vlm_file in vlm_param_list:
                 if args['prompt_file'] is not None and len(args['prompt_file']) > 0:
                     vlm_file['media'] = model_utils.resolve_media_file_path(vlm_file.get("media"), args['prompt_file'][0])
                 vlm_file_list.append(vlm_file)
-    else:
-        vlm_file_list.append(output_data_list)
+    else: vlm_file_list.append(output_data_list)
     return vlm_file_list