[GHA] Replaced isual_language_chat_sample-ubuntu-minicpm_v2_6 job (openvinotoolkit#1909)

Moved tests to Linux/Windows pipelines

---------

Co-authored-by: Vladimir Zlobin <vladimir.zlobin@intel.com>

Author: mryzhov

.github/workflows/causal_lm_cpp.yml

@@ -89,100 +89,6 @@ jobs:
           diff pred2.txt ref.txt
           echo "Chat sample python" passed
 
-  visual_language_chat_sample-ubuntu-minicpm_v2_6:
-    runs-on: ubuntu-22.04-16-cores
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-      - uses: ./.github/actions/install_openvino
-        with:
-          ov_link: ${{ env.l_u22_ov_link }}
-      - uses: ./.github/actions/build_app
-        with:
-          build_target: 'visual_language_chat benchmark_vlm py_openvino_genai'
-      - uses: ./.github/actions/install_python_deps
-      - name: Download and convert tiny-random-minicpmv-2_6 model and an image
-        run: |
-          source ./ov/setupvars.sh
-          optimum-cli export openvino -m katuni4ka/tiny-random-minicpmv-2_6 tiny-random-minicpmv-2_6 --trust-remote-code --task image-text-to-text
-          mkdir images
-      - name: Generate images - tiny-random-minicpmv-2_6
-        shell: python
-        run: |
-          from PIL import Image
-          import numpy as np
-          import requests
-          res = 28, 28
-          lines = np.arange(res[0] * res[1] * 3, dtype=np.uint8) % 255
-          lines = lines.reshape([*res, 3])
-          lines_image = Image.fromarray(lines)
-          lines_image.save("images/lines.png")
-          cat = Image.open(requests.get("https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11", stream=True).raw).convert('RGB')
-          cat.save("images/cat.png")
-      - name: Run visual_language_chat C++ sample - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./build/samples/cpp/visual_language_chat/visual_language_chat ./tiny-random-minicpmv-2_6/ ./images/
-          <<< $'Describe the images?' | tee cpp.txt
-        timeout-minutes: 2
-      - name: Run benchmark_vlm C++ sample - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./build/samples/cpp/visual_language_chat/benchmark_vlm  -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3
-        timeout-minutes: 2
-      - name: Run visual_language_chat Python sample - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./samples/python/visual_language_chat/visual_language_chat.py ./tiny-random-minicpmv-2_6/ ./images/
-          <<< $'Describe the images?' | tee py.txt
-        env:
-          PYTHONPATH: "./build/"
-      - name: Run benchmark_vlm Python sample - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./samples/python/visual_language_chat/benchmark_vlm.py -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3
-        env:
-          PYTHONPATH: "./build/"
-      - name: Encode cpp.txt with Python encoding instead of terminal one
-        shell: python
-        run: |
-          with open("cpp.txt", "rb") as f:
-            content = f.read().decode("utf-8", "replace")
-          with open("cpp.txt", "wb") as f:
-            f.write(content.encode("utf-8"))
-      - run: diff cpp.txt py.txt
-      - name: Run visual_language_chat C++ sample with 2 prompts - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./build/samples/cpp/visual_language_chat/visual_language_chat ./tiny-random-minicpmv-2_6/ ./images/cat.png
-          <<< $'What is unusual on this image?\nGo on.' | tee cpp2.txt
-        timeout-minutes: 2
-      - name: Run visual_language_chat Python sample with 2 prompts - tiny-random-minicpmv-2_6
-        run: >
-          set -o pipefail
-          && source ./ov/setupvars.sh
-          && ./samples/python/visual_language_chat/visual_language_chat.py ./tiny-random-minicpmv-2_6/ ./images/cat.png
-          <<< $'What is unusual on this image?\nGo on.' | tee py2.txt
-        env:
-          PYTHONPATH: "./build/"
-      - name: Encode cpp2.txt with Python encoding instead of terminal one
-        shell: python
-        run: |
-          with open("cpp2.txt", "rb") as f:
-            content = f.read().decode("utf-8", "replace")
-          with open("cpp2.txt", "wb") as f:
-            f.write(content.encode("utf-8"))
-      - run: diff cpp2.txt py2.txt
-
   cpp-continuous-batching-ubuntu:
     runs-on: ubuntu-22.04-8-cores
     defaults:
@@ -320,8 +226,7 @@ jobs:
 
   Overall_Status:
     name: ci/gha_overall_status_causal_lm
-    needs: [cpp-chat_sample-ubuntu, cpp-continuous-batching-ubuntu,
-            visual_language_chat_sample-ubuntu-minicpm_v2_6, cpp-continuous-batching-windows, cpp-continuous-batching-macos]
+    needs: [cpp-chat_sample-ubuntu, cpp-continuous-batching-ubuntu, cpp-continuous-batching-windows, cpp-continuous-batching-macos]
     if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:

tests/python_tests/samples/conftest.py

@@ -77,6 +77,10 @@
         "name": "OpenVINO/LCM_Dreamshaper_v7-int8-ov",
         "convert_args": []
     },
+    "tiny-random-minicpmv-2_6": {
+        "name": "katuni4ka/tiny-random-minicpmv-2_6",
+        "convert_args": ['--trust-remote-code', "--task", "image-text-to-text"]
+    },
     "InternVL2-1B": {
         "name": "OpenGVLab/InternVL2-1B",
         "convert_args": ['--trust-remote-code']
@@ -92,8 +96,8 @@
     "adapter_model.safetensors": "https://huggingface.co/smangrul/tinyllama_lora_sql/resolve/main/adapter_model.safetensors",
     "monalisa.jpg": "https://llava-vl.github.io/static/images/monalisa.jpg",
     "soulcard.safetensors": "https://civitai.com/api/download/models/72591",
-    "image.png": "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png",
-    "mask_image.png": "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
+    "images/image.png": "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png",
+    "mask_image.png": "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png",
 }
 
 SAMPLES_PY_DIR = os.environ.get("SAMPLES_PY_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../samples/python")))
@@ -198,7 +202,8 @@ def download_test_content(request):
     file_url = TEST_FILES[file_name]
     file_path = os.path.join(test_data, file_name)
     if not os.path.exists(file_path):
-        logger.info(f"Downloading test content from {file_url}...")
+        logger.info(f"Downloading test content from {file_url} to {file_path}...")
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
         response = requests.get(file_url, stream=True)
         response.raise_for_status()
         with open(file_path, 'wb') as f:
@@ -215,6 +220,35 @@ def download_test_content(request):
             os.remove(file_path)
 
 
+@pytest.fixture(scope="session")
+def generate_test_content(request):
+    """Generate an image of lines and return the file path."""
+    
+    test_data = request.config.cache.get("TEST_DATA", None)
+    
+    file_name = request.param
+    file_path = os.path.join(test_data, file_name)
+    if not os.path.exists(file_path):
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+        from PIL import Image
+        import numpy as np
+        res = 28, 28
+        lines = np.arange(res[0] * res[1] * 3, dtype=np.uint8) % 255
+        lines = lines.reshape([*res, 3])
+        lines_image = Image.fromarray(lines)
+        lines_image.save(file_path)
+        logger.info(f"Generated test content {file_path}")
+    else:
+        logger.info(f"Test content already exists at {file_path}")
+    yield file_path
+    # Cleanup the test content after tests
+    if os.environ.get("CLEANUP_CACHE", "false").lower() == "true":
+        if os.path.exists(file_path):
+            logger.info(f"Removing test content: {file_path}")
+            os.remove(file_path)
+
+
 @pytest.fixture(scope="module", autouse=True)
 def run_gc_after_test():
     """

tests/python_tests/samples/test_benchmark_image_gen.py

@@ -23,7 +23,7 @@ class TestBenchmarkImageGen:
     @pytest.mark.parametrize(
         "download_test_content, download_mask_image",
         [
-            pytest.param("image.png", "mask_image.png"),
+            pytest.param("images/image.png", "mask_image.png"),
         ], indirect=["download_test_content", "download_mask_image"],
     )
     def test_sample_benchmark_image_gen(self, download_model, pipeline_type, prompt, download_test_content, download_mask_image):

tests/python_tests/samples/test_benchmark_vlm.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+import sys
+
+from conftest import SAMPLES_PY_DIR, SAMPLES_CPP_DIR
+from test_utils import run_sample
+
+class TestBenchmarkVLM:
+    @pytest.mark.vlm
+    @pytest.mark.samples
+    @pytest.mark.parametrize(
+        "convert_model, download_test_content",
+        [
+            pytest.param("tiny-random-minicpmv-2_6", "images/image.png"),
+        ],
+        indirect=["convert_model", "download_test_content"],
+    )
+    def test_sample_benchmark_vlm(self, convert_model, download_test_content):
+        num_iter = "3"
+        # Run C++ benchmark sample
+        benchmark_sample = os.path.join(SAMPLES_CPP_DIR, 'benchmark_vlm')
+        benchmark_cpp_command = [benchmark_sample, "-m" , convert_model, "-i", download_test_content, "-n", num_iter]
+        run_sample(benchmark_cpp_command)
+        
+        # Run Python benchmark sample
+        benchmark_script = os.path.join(SAMPLES_PY_DIR, 'visual_language_chat/benchmark_vlm.py')
+        benchmark_py_command = [sys.executable, benchmark_script, "-m" , convert_model, "-i", download_test_content, "-n", num_iter]
+        run_sample(benchmark_py_command)

tests/python_tests/samples/test_image2image.py

@@ -18,7 +18,7 @@ class TestImage2Image:
         ],
         indirect=["download_model"],
     )
-    @pytest.mark.parametrize("download_test_content", ["image.png"], indirect=True)
+    @pytest.mark.parametrize("download_test_content", ["images/image.png"], indirect=True)
     def test_sample_image2image(self, download_model, prompt, download_test_content):
         # Run Python sample
         py_script = os.path.join(SAMPLES_PY_DIR, "image_generation/image2image.py")

tests/python_tests/samples/test_inpainting.py

@@ -23,7 +23,7 @@ class TestInpainting:
     @pytest.mark.parametrize(
         "download_test_content, download_mask_image",
         [
-            pytest.param("image.png", "mask_image.png"),
+            pytest.param("images/image.png", "mask_image.png"),
         ],
         indirect=["download_test_content", "download_mask_image"],
     )

tests/python_tests/samples/test_visual_language_chat.py

@@ -12,29 +12,54 @@ class TestVisualLanguageChat:
     @pytest.mark.vlm
     @pytest.mark.samples
     @pytest.mark.parametrize(
-        "convert_model, sample_args",
+        "convert_model, download_test_content, questions",
         [
-            pytest.param("llava-1.5-7b-hf", 'Who drew this painting?\nWhen did the painter live?'),
-            pytest.param("llava-v1.6-mistral-7b-hf", 'Who drew this painting?\nWhen did the painter live?'),
-            pytest.param("InternVL2-1B", 'Who drew this painting?\nWhen did the painter live?'),
-            pytest.param("Qwen2-VL-2B-Instruct", 'Who drew this painting?\nWhen did the painter live?'),
+            pytest.param("llava-1.5-7b-hf", "monalisa.jpg", 'Who drew this painting?\nWhen did the painter live?'),
+            pytest.param("llava-v1.6-mistral-7b-hf", "monalisa.jpg", 'Who drew this painting?\nWhen did the painter live?'),
+            pytest.param("InternVL2-1B", "monalisa.jpg", 'Who drew this painting?\nWhen did the painter live?'),
+            pytest.param("Qwen2-VL-2B-Instruct", "monalisa.jpg", 'Who drew this painting?\nWhen did the painter live?'),
+            pytest.param("tiny-random-minicpmv-2_6", "images/image.png", 'What is unusual on this image?\nGo on.')
         ],
-        indirect=["convert_model"],
+        indirect=["convert_model", "download_test_content"],
     )
-    @pytest.mark.parametrize("download_test_content", ["monalisa.jpg"], indirect=True)
-    def test_sample_visual_language_chat(self, request, convert_model, download_test_content, sample_args):
+    def test_sample_visual_language_chat(self, request, convert_model, download_test_content, questions):
         model_name = request.node.callspec.params['convert_model']
 
         # Test Python sample
         py_script = os.path.join(SAMPLES_PY_DIR, "visual_language_chat/visual_language_chat.py")
         py_command = [sys.executable, py_script, convert_model, download_test_content]
-        py_result = run_sample(py_command, sample_args)
+        py_result = run_sample(py_command, questions)
 
         # Test CPP sample
         cpp_sample = os.path.join(SAMPLES_CPP_DIR, 'visual_language_chat')
         cpp_command =[cpp_sample, convert_model, download_test_content]
-        cpp_result = run_sample(cpp_command, sample_args)
+        cpp_result = run_sample(cpp_command, questions)
 
         # Compare results
         if model_name != "Qwen2-VL-2B-Instruct": # "Skipping result comparison for Qwen2-VL-2B-Instruct due to CVS-164144"
             assert py_result.stdout == cpp_result.stdout, f"Results should match"
+
+    @pytest.mark.vlm
+    @pytest.mark.samples
+    @pytest.mark.parametrize(
+        "convert_model, questions",
+        [
+            pytest.param("tiny-random-minicpmv-2_6", 'Describe the images?'),
+        ],
+        indirect=["convert_model"],
+    )
+    @pytest.mark.parametrize("download_test_content", ["images/image.png"], indirect=True)
+    @pytest.mark.parametrize("generate_test_content", ["images/lines.png"], indirect=True)
+    def test_sample_visual_language_chat_images(self, convert_model, download_test_content, generate_test_content, questions):
+        # Test Python sample
+        py_script = os.path.join(SAMPLES_PY_DIR, "visual_language_chat/visual_language_chat.py")
+        py_command = [sys.executable, py_script, convert_model, os.path.dirname(generate_test_content)]
+        py_result = run_sample(py_command, questions)
+
+        # Test CPP sample
+        cpp_sample = os.path.join(SAMPLES_CPP_DIR, 'visual_language_chat')
+        cpp_command =[cpp_sample, convert_model, os.path.dirname(generate_test_content)]
+        cpp_result = run_sample(cpp_command, questions)
+
+        # Compare results
+        assert py_result.stdout == cpp_result.stdout, f"Results should match"