test

Author: sbalandi

.github/workflows/linux.yml

@@ -616,11 +616,14 @@ jobs:
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }}
             timeout: 30
           - name: 'WWB tests'
-            cmd: 'python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"'
+            cmd: |
+              python -m pip install ${{ github.workspace }}/install/wheels/whowhatbench
+              python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
           - name: 'WWB tests (nanollava)'
             cmd: |
+              python -m pip install ${{ github.workspace }}/install/wheels/whowhatbench
               python -m pip install transformers==4.48.0
               python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
@@ -667,7 +670,7 @@ jobs:
         if: ${{ matrix.test.run_condition }}
         uses: ./src/.github/actions/install_wheel
         with:
-          packages: "openvino;openvino_tokenizers[transformers];openvino_genai;whowhatbench"
+          packages: "openvino;openvino_tokenizers[transformers];openvino_genai"
           requirements_files: "${{ env.SRC_DIR }}/tests/python_tests/requirements.txt"
           local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels
 
@@ -871,7 +874,7 @@ jobs:
         uses: ./src/.github/actions/install_wheel
         with:
           packages: "openvino;openvino_tokenizers[transformers];openvino_genai[testing]"
-          requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt;${{ env.SRC_DIR }}/tools/llm_bench/requirements.txt"
+          requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt"
           local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels
 
       - name: gtests unit tests
@@ -893,6 +896,7 @@ jobs:
         if: ${{ fromJSON(needs.smart_ci.outputs.affected_components).llm_bench }}
         run: |
           source ${{ env.INSTALL_DIR }}/setupvars.sh
+          python -m pip install -r ${{ env.SRC_DIR }}/tools/llm_bench/requirements.txt
           python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_tools_llm_benchmark.py -m "samples"
         env:
           SAMPLES_PY_DIR: "${{ env.SRC_DIR }}/tools"

.github/workflows/mac.yml

@@ -468,11 +468,14 @@ jobs:
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }}
             timeout: 30
           - name: 'WWB tests'
-            cmd: 'python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"'
+            cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
+              python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
           - name: 'WWB tests (nanollava)'
             cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
               python -m pip install transformers==4.48.0
               python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}

.github/workflows/manylinux_2_28.yml

@@ -539,11 +539,14 @@ jobs:
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }}
             timeout: 30
           - name: 'WWB tests'
-            cmd: 'python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"'
+            cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
+              python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
           - name: 'WWB tests (nanollava)'
             cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
               python -m pip install transformers==4.48.0
               python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}

.github/workflows/windows.yml

@@ -704,11 +704,14 @@ jobs:
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }}
             timeout: 30
           - name: 'WWB tests'
-            cmd: 'python -m pytest -s -v tools/who_what_benchmark/tests -m "not nanollava"'
+            cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
+              python -m pytest -s -v tools/who_what_benchmark/tests -m "not nanollava"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
           - name: 'WWB tests (nanollava)'
             cmd: |
+              python -m pip install ${{ env.INSTALL_DIR }}/wheels/whowhatbench
               python -m pip install transformers==4.48.0
               python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}

tools/who_what_benchmark/whowhatbench/model_loaders.py

@@ -185,7 +185,7 @@ def load_text_hf_pipeline(model_id, device, **kwargs):
     else:
         try:
             model = AutoModelForCausalLM.from_pretrained(
-                model_id, trust_remote_code=False, device_map=device.lower(), **model_kwargs
+                model_id, trust_remote_code=True, device_map=device.lower(), **model_kwargs
             )
         except Exception:
             model = AutoModelForCausalLM.from_pretrained(

tools/who_what_benchmark/whowhatbench/wwb.py

@@ -838,6 +838,7 @@ def print_rag_results(evaluator):
 def main():
     args = parse_args()
     check_args(args)
+    print("print ci")
 
     version_str = f'openvino runtime version: {ov.get_version()}'
     if args.genai: