diff --git a/tests/test_omnimat_data_url.py b/tests/test_omnimat_data_url.py new file mode 100644 index 000000000..64978e625 --- /dev/null +++ b/tests/test_omnimat_data_url.py @@ -0,0 +1,37 @@ +import pandas as pd + +from vlmeval.dataset import omnimat as omnimat_module +from vlmeval.dataset.omnimat import OmniMat + +EXPECTED_URLS = { + 'OmniMat_QA': ( + 'https://huggingface.co/datasets/' + 'Summer12138/OmniMat1K-VLMEvalKit/resolve/main/' + 'vlmevalkit/OmniMat_QA.tsv' + ), + 'OmniMat_CAL': ( + 'https://huggingface.co/datasets/' + 'Summer12138/OmniMat1K-VLMEvalKit/resolve/main/' + 'vlmevalkit/OmniMat_CAL.tsv' + ), +} + + +def test_omnimat_uses_published_tsv_urls(): + assert OmniMat.DATASET_URL == EXPECTED_URLS + + +def test_omnimat_downloads_published_tsv_when_local_data_is_missing(tmp_path, monkeypatch): + monkeypatch.setattr(omnimat_module, 'LMUDataRoot', lambda: str(tmp_path)) + dataset = object.__new__(OmniMat) + downloaded = pd.DataFrame( + [{'index': '1', 'category_id': '1', 'id': '2', 'question': 'test'}] + ) + called = [] + dataset.prepare_tsv = lambda url: called.append(url) or downloaded.copy() + + data = dataset.load_data('OmniMat_QA') + + assert called == [EXPECTED_URLS['OmniMat_QA']] + assert data.iloc[0]['category_id'] == '01' + assert data.iloc[0]['id'] == '002' diff --git a/vlmeval/dataset/omnimat.py b/vlmeval/dataset/omnimat.py index 4e3e3892c..c2037140a 100644 --- a/vlmeval/dataset/omnimat.py +++ b/vlmeval/dataset/omnimat.py @@ -22,8 +22,16 @@ class OmniMat(ImageBaseDataset): DEFAULT_JUDGE = 'gemini-2.5-flash' DATASET_URL = { - 'OmniMat_QA': '', - 'OmniMat_CAL': '', + 'OmniMat_QA': ( + 'https://huggingface.co/datasets/' + 'Summer12138/OmniMat1K-VLMEvalKit/resolve/main/' + 'vlmevalkit/OmniMat_QA.tsv' + ), + 'OmniMat_CAL': ( + 'https://huggingface.co/datasets/' + 'Summer12138/OmniMat1K-VLMEvalKit/resolve/main/' + 'vlmevalkit/OmniMat_CAL.tsv' + ), } QA_PROMPT = ( @@ -66,6 +74,16 @@ def load_data(self, dataset): _normalize_ids(data) return data + url = self.DATASET_URL.get(dataset) + if url: + data = self.prepare_tsv(url) + if 'image' in data: + data['image'] = data['image'].fillna('') + if 'image_path' in data: + data['image_path'] = data['image_path'].fillna('') + _normalize_ids(data) + return data + raise FileNotFoundError( f'{dataset}.tsv was not found. Run `python scripts/convert_omnimat.py` ' 'from the repository root first, or place the TSV under LMUData.'