Skip to content

Commit 8a1be33

Browse files
sbalandisbalandi
committed
test11
1 parent 8b5e4b7 commit 8a1be33

File tree

10 files changed

+54
-30
lines changed

10 files changed

+54
-30
lines changed

‎.github/workflows/linux.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,6 @@ jobs:
612612
env_hf_cache: /mount/caches/pytest/datasets_test
613613
cmd: |
614614
echo "HF_HUB_DOWNLOAD_TIMEOUT=$HF_HUB_DOWNLOAD_TIMEOUT"
615-
echo "HF_DATASETS_CACHE=$HF_DATASETS_CACHE"
616615
python -m pip install transformers==4.48.0
617616
python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
618617
run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
@@ -667,7 +666,6 @@ jobs:
667666
if: ${{ matrix.test.run_condition }}
668667
run: |
669668
if [ -n "${{ matrix.test.env_hf_cache }}" ]; then
670-
export HF_DATASETS_CACHE="${{ matrix.test.env_hf_cache }}"
671669
export HF_HUB_DOWNLOAD_TIMEOUT=100
672670
fi
673671
${{ matrix.test.cmd }}

‎.github/workflows/manylinux_2_28.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,6 @@ jobs:
588588
if: ${{ matrix.test.run_condition }}
589589
run: |
590590
if [ -n "${{ matrix.test.env_hf_cache }}" ]; then
591-
export HF_DATASETS_CACHE="${{ matrix.test.env_hf_cache }}"
592591
export HF_HUB_DOWNLOAD_TIMEOUT=100
593592
fi
594593
${{ matrix.test.cmd }}

‎.github/workflows/windows.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -755,7 +755,6 @@ jobs:
755755
if: ${{ matrix.test.run_condition }}
756756
run: |
757757
if ("${{ matrix.test.env_hf_cache }}" -ne "") {
758-
$env:HF_DATASETS_CACHE="${{ matrix.test.env_hf_cache }}"
759758
$env:HF_HUB_DOWNLOAD_TIMEOUT=100
760759
}
761760
${{ matrix.test.cmd }}

‎tools/who_what_benchmark/tests/conftest.py‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,6 @@
5858
"tiny-random-flux-fill": {"name": "katuni4ka/tiny-random-flux-fill", "convert_args": []},
5959
}
6060

61-
DATASETS: Dict[str, Dict[str, Any]] = {
62-
"microsoft/ms_marco": {
63-
"kwargs": {"split": "test", "name": "v2.1"}
64-
},
65-
"paint-by-inpaint/PIPE": {
66-
"kwargs": {"split": "test"}
67-
},
68-
"phiyodr/InpaintCOCO": {
69-
"kwargs": {"split": "test"}
70-
},
71-
"ucla-contextual/contextual_test": {
72-
"kwargs": {"split": "test"}
73-
}
74-
}
7561

7662
def get_ov_cache_converted_models_dir():
7763
return get_ov_cache_dir() / "converted_models"
@@ -116,12 +102,6 @@ def convert(temp_path: Path) -> None:
116102

117103
@pytest.fixture(scope="session", autouse=True)
118104
def module_teardown():
119-
lock_file_path = os.path.join(os.environ.get("HF_DATASETS_CACHE", "."), "hf_download.lock")
120-
lock = FileLock(lock_file_path)
121-
with lock:
122-
for name, info in DATASETS.items():
123-
datasets.load_dataset(name, **info["kwargs"])
124-
125105
ov_cache_dir = get_ov_cache_dir()
126106
ov_cache_converted_dir = get_ov_cache_converted_models_dir()
127107
logger.info(f"Creating directories: {ov_cache_converted_dir}")
@@ -140,7 +120,7 @@ def module_teardown():
140120

141121
def run_wwb(args, env=None):
142122
command = ["wwb"] + args
143-
base_env = {"TRANSFORMERS_VERBOSITY": "debug", "PYTHONIOENCODING": "utf-8", "HF_DATASETS_OFFLINE": "1", **os.environ}
123+
base_env = {"TRANSFORMERS_VERBOSITY": "debug", "PYTHONIOENCODING": "utf-8", **os.environ}
144124
if env:
145125
base_env.update(env)
146126
try:

‎tools/who_what_benchmark/whowhatbench/check_llava_next_hf_opt_pictures.txt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
Video Similarity evaluation: 10it [04:12, 25.30s/it]
2+
INFO:__main__:Metrics for model: /home/labuser/work/notebook/openvino.genai/models/ltx_hf
3+
INFO:__main__: similarity
4+
0 0.926587
5+
INFO:__main__:=======================================================================================================
6+
INFO:__main__:Top-1 example:
7+
INFO:__main__:similarity 0.872807
8+
per_frame [0.863993, 0.8697232, 0.87982726, 0.88633025, 0.8948775, 0.88551706, 0.867858, 0.87493384, 0.89429426, 0.8620807, 0.85930896, 0.8789909, 0.86685336, 0.8588746, 0.85176444, 0.8771286, 0.8655051, 0.860549, 0.8855689, 0.8653542, 0.87040496, 0.8642998, 0.87230325, 0.88042855, 0.8834051]
9+
prompts cowboy running in slow motion in a field
10+
source_model reference/video_0
11+
optimized_model /home/labuser/work/notebook/openvino.genai/out_hf/target/video_0
12+
Name: 0, dtype: object
13+
INFO:__main__:=======================================================================================================
14+
INFO:__main__:Top-2 example:
15+
INFO:__main__:similarity 0.887368
16+
per_frame [0.8871872, 0.8869467, 0.85796607, 0.8967118, 0.88550436, 0.89251703, 0.87551755, 0.9005059, 0.88514113, 0.88472545, 0.8837644, 0.8999299, 0.8868808, 0.8852254, 0.8904059, 0.88806957, 0.8790874, 0.88376653, 0.88198626, 0.8739811, 0.8960515, 0.8943306, 0.9037218, 0.8883736, 0.8959048]
17+
prompts octopus figure skating, cartoon
18+
source_model reference/video_7
19+
optimized_model /home/labuser/work/notebook/openvino.genai/out_hf/target/video_7
20+
Name: 7, dtype: object
21+
INFO:__main__:=======================================================================================================
22+
INFO:__main__:Top-3 example:
23+
INFO:__main__:similarity 0.898533
24+
per_frame [0.8846318, 0.89364123, 0.89606816, 0.90520835, 0.89990604, 0.90730584, 0.9084669, 0.90380013, 0.91040766, 0.89905477, 0.8743136, 0.9103744, 0.88250816, 0.8999705, 0.90139693, 0.90683913, 0.9051002, 0.8936784, 0.88980436, 0.8977965, 0.8970734, 0.89419603, 0.9076474, 0.89829046, 0.89583224]
25+
prompts fight naruto vs saske
26+
source_model reference/video_3
27+
optimized_model /home/labuser/work/notebook/openvino.genai/out_hf/target/video_3
28+
Name: 3, dtype: object
29+
INFO:__main__:=======================================================================================================
30+
INFO:__main__:Top-4 example:
31+
INFO:__main__:similarity 0.92063
32+
per_frame [0.92601585, 0.91334426, 0.9104391, 0.9115267, 0.9206182, 0.9128696, 0.9165244, 0.9295911, 0.90867174, 0.9169818, 0.9296118, 0.9249383, 0.9260864, 0.91794574, 0.92485356, 0.9308256, 0.9226842, 0.90882367, 0.92773736, 0.89623374, 0.92181456, 0.92432696, 0.9271541, 0.9267959, 0.9393321]
33+
prompts slow motion, hydrogen bond energy, atom, 4k, cinematic -gs 24 -motion 2 -ar 16:9 -fps 24
34+
source_model reference/video_2
35+
optimized_model /home/labuser/work/notebook/openvino.genai/out_hf/target/video_2
36+
Name: 2, dtype: object
37+
INFO:__main__:=======================================================================================================
38+
INFO:__main__:Top-5 example:
39+
INFO:__main__:similarity 0.92158
40+
per_frame [0.91554844, 0.9118484, 0.9261645, 0.9216336, 0.9255843, 0.92744607, 0.9236192, 0.91328067, 0.92231536, 0.9318491, 0.9139346, 0.9283084, 0.9100289, 0.9230741, 0.91734266, 0.9194629, 0.9174213, 0.92608166, 0.91852343, 0.92541015, 0.9288535, 0.92535686, 0.92713904, 0.9156982, 0.9235669]
41+
prompts indian womens wahsing clothes at river side
42+
source_model reference/video_6
43+
optimized_model /home/labuser/work/notebook/openvino.genai/out_hf/target/video_6
44+
Name: 6, dtype: object
45+
46+
47+
0 0.839385
48+
0 0.898635

‎tools/who_what_benchmark/whowhatbench/embeddings_evaluator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def prepare_default_data(num_samples=None):
2222
NUM_SAMPLES = num_samples if num_samples else 24
2323
set_seed(42)
2424
default_dataset = datasets.load_dataset(
25-
DATASET_NAME, 'v2.1', split="test", streaming=True
25+
DATASET_NAME, 'v2.1', split="test", streaming=True, download_config=datasets.DownloadConfig(max_retries=10)
2626
).shuffle(42).take(NUM_SAMPLES)
2727
return default_dataset.map(
2828
lambda x: {'passages': x['passages']['passage_text']}, remove_columns=default_dataset.column_names

‎tools/who_what_benchmark/whowhatbench/im2im_evaluator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def prepare_default_data(num_samples=None):
2525
NUM_SAMPLES = 10 if num_samples is None else num_samples
2626
set_seed(42)
2727
default_dataset = datasets.load_dataset(
28-
DATASET_NAME, split="test", streaming=True
28+
DATASET_NAME, split="test", streaming=True, download_config=datasets.DownloadConfig(max_retries=10)
2929
).filter(lambda example: example["Instruction_VLM-LLM"] != "").take(NUM_SAMPLES)
3030
return default_dataset.map(
3131
lambda x: preprocess_fn(x), remove_columns=default_dataset.column_names

‎tools/who_what_benchmark/whowhatbench/inpaint_evaluator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def prepare_default_data(num_samples=None):
4141
NUM_SAMPLES = 10 if num_samples is None else num_samples
4242
set_seed(42)
4343
default_dataset = datasets.load_dataset(
44-
DATASET_NAME, split="test", streaming=True,
44+
DATASET_NAME, split="test", streaming=True, download_config=datasets.DownloadConfig(max_retries=10),
4545
).filter(lambda example: example["inpaint_caption"] != "").take(NUM_SAMPLES)
4646
return default_dataset.map(
4747
lambda x: preprocess_fn(x), remove_columns=default_dataset.column_names

‎tools/who_what_benchmark/whowhatbench/reranking_evaluator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def prepare_default_data(num_samples=None):
4040
NUM_SAMPLES = num_samples if num_samples else 24
4141
set_seed(42)
4242
default_dataset = datasets.load_dataset(
43-
DATASET_NAME, 'v2.1', split="test", streaming=True
43+
DATASET_NAME, 'v2.1', split="test", streaming=True, download_config=datasets.DownloadConfig(max_retries=10)
4444
).shuffle(42).take(NUM_SAMPLES)
4545
return default_dataset.map(
4646
lambda x: preprocess_fn(x), remove_columns=default_dataset.column_names

‎tools/who_what_benchmark/whowhatbench/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def prepare_default_data_image(num_samples=None):
160160
NUM_SAMPLES = 24 if num_samples is None else num_samples
161161
set_seed(42)
162162
default_dataset = datasets.load_dataset(
163-
DATASET_NAME, split="test", streaming=True
163+
DATASET_NAME, split="test", streaming=True, download_config=datasets.DownloadConfig(max_retries=10),
164164
).shuffle(42).take(NUM_SAMPLES)
165165
return default_dataset.map(
166166
lambda x: preprocess_fn(x), remove_columns=default_dataset.column_names

0 commit comments

Comments
 (0)