[evaluation] fix comments for Video-mme dataset support

This commit fixes comments for Video-mme dataset support

TICO-DCO-1.0-Signed-off-by:  Evgenii Maltsev <e.maltsev@samsung.com>

Author: Torrero

test/quantization/recipes/test_video_mme_evaluation.py

@@ -90,11 +90,9 @@ def fake_print_lmms_eval_results(results):
                 processor=MagicMock(),
                 device="cpu",
                 use_cache="/tmp/cache",
-                cache_dir="/tmp/hf_cache",
             )
 
         self.assertEqual(captured["use_cache"], "/tmp/cache")
-        self.assertEqual(captured["cache_dir"], "/tmp/hf_cache")
 
 
 class TestLmmsEvalUtils(unittest.TestCase):
@@ -109,22 +107,10 @@ def test_build_model_args_infers_qwen3_vl(self):
         model_name_str = "Qwen/Qwen3-VL-2B-Instruct"
         model.config._name_or_path = model_name_str
 
-        processor = MagicMock()
-        processor.tokenizer.name_or_path = model_name_str
-
-        model_name, model_args = _build_model_args(
-            model, processor, device="cuda", batch_size=2, max_new_tokens=16
-        )
+        model_name, model_args = _build_model_args(model)
 
         self.assertEqual(model_name, "qwen3_vl")
         self.assertEqual(model_args["pretrained"], model_name_str)
-        # Only 'pretrained' is in model_args; batch_size, device,
-        # max_new_tokens, and tokenizer are NOT (they'd cause errors
-        # with lmms-eval model constructors like Qwen3_VL).
-        self.assertNotIn("batch_size", model_args)
-        self.assertNotIn("device", model_args)
-        self.assertNotIn("max_new_tokens", model_args)
-        self.assertNotIn("tokenizer", model_args)
 
     def test_build_model_args_passes_max_num_frames(self):
         """_build_model_args should include max_num_frames in model_args."""
@@ -134,14 +120,8 @@ def test_build_model_args_passes_max_num_frames(self):
         type(model).__name__ = "Qwen3VLForConditionalGeneration"
         model.config._name_or_path = "Qwen/Qwen3-VL-2B-Instruct"
 
-        processor = MagicMock()
-
         model_name, model_args = _build_model_args(
             model,
-            processor,
-            device="cuda",
-            batch_size=1,
-            max_new_tokens=16,
             max_num_frames=5,
         )
 
@@ -164,7 +144,7 @@ def test_get_custom_tasks_dir_finds_lmms_tasks(self):
 
         tasks_dir = _get_custom_tasks_dir()
         self.assertIsNotNone(tasks_dir)
-        self.assertTrue(tasks_dir.endswith("lmms_tasks"))
+        self.assertTrue(tasks_dir.endswith("lmms_tasks"))  # type: ignore[union-attr]
 
     def test_print_results_fallback(self):
         """Fallback printer should handle float and non-float values."""
@@ -205,16 +185,18 @@ def test_limit_1_downloads_1_chunk(self):
         self.assertIn("videos_chunked_01.zip", patterns)
         self.assertNotIn("videos_chunked_02.zip", patterns)
 
-    def test_limit_41_downloads_2_chunks(self):
-        """A limit of 41 (> _SAMPLES_PER_CHUNK) should download 2 chunks."""
+    def test_limit_61_downloads_3_chunks(self):
+        """A limit of 61 (> 2*_SAMPLES_PER_CHUNK) should download 3 chunks."""
         from tico.quantization.evaluation.lmms_eval_utils import (
             _compute_video_chunk_patterns,
         )
 
-        patterns = _compute_video_chunk_patterns(limit=41)
+        # _SAMPLES_PER_CHUNK = 30, so ceil(61/30) = 3 chunks
+        patterns = _compute_video_chunk_patterns(limit=61)
         self.assertIn("videos_chunked_01.zip", patterns)
         self.assertIn("videos_chunked_02.zip", patterns)
-        self.assertNotIn("videos_chunked_03.zip", patterns)
+        self.assertIn("videos_chunked_03.zip", patterns)
+        self.assertNotIn("videos_chunked_04.zip", patterns)
 
     def test_limit_none_downloads_all_chunks(self):
         """No limit (None) should download all 20 chunks."""
@@ -268,6 +250,144 @@ def test_non_videomme_repo_passes_through(self):
             pass
 
 
+class TestGetDownloadedVideommeChunks(unittest.TestCase):
+    """Tests for _get_downloaded_videomme_chunks."""
+
+    def test_returns_empty_set_when_no_cache(self):
+        """Should return empty set when HF cache dir doesn't exist."""
+        # With a non-existent HF_HOME, should return empty set
+        import os
+
+        from tico.quantization.evaluation.lmms_eval_utils import (
+            _get_downloaded_videomme_chunks,
+        )
+
+        original_hf = os.environ.get("HF_HOME")
+        try:
+            os.environ["HF_HOME"] = "/tmp/nonexistent_hf_cache_12345"
+            result = _get_downloaded_videomme_chunks()
+            self.assertIsInstance(result, set)
+            self.assertEqual(len(result), 0)
+        finally:
+            if original_hf is not None:
+                os.environ["HF_HOME"] = original_hf
+            else:
+                os.environ.pop("HF_HOME", None)
+
+    def test_finds_chunks_in_fake_cache(self):
+        """Should find chunk zips in a fake cache directory."""
+        import os
+        import tempfile
+
+        from tico.quantization.evaluation.lmms_eval_utils import (
+            _get_downloaded_videomme_chunks,
+        )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Create the expected directory structure
+            repo_dir = os.path.join(tmpdir, "hub", "datasets--lmms-lab--Video-MME")
+            snap_dir = os.path.join(repo_dir, "snapshots", "abc123")
+            os.makedirs(snap_dir)
+
+            # Create fake chunk files
+            for name in ["videos_chunked_01.zip", "videos_chunked_02.zip"]:
+                with open(os.path.join(snap_dir, name), "w") as f:
+                    f.write("fake zip content")
+
+            # Create a non-chunk file that should be ignored
+            with open(os.path.join(snap_dir, "subtitle.zip"), "w") as f:
+                f.write("fake subtitle")
+
+            original_hf = os.environ.get("HF_HOME")
+            try:
+                os.environ["HF_HOME"] = tmpdir
+                result = _get_downloaded_videomme_chunks()
+                self.assertIn("videos_chunked_01.zip", result)
+                self.assertIn("videos_chunked_02.zip", result)
+                self.assertNotIn("subtitle.zip", result)
+                self.assertEqual(len(result), 2)
+            finally:
+                if original_hf is not None:
+                    os.environ["HF_HOME"] = original_hf
+                else:
+                    os.environ.pop("HF_HOME", None)
+
+
+class TestEnsureVideommeChunksDownloaded(unittest.TestCase):
+    """Tests for _ensure_videomme_chunks_downloaded."""
+
+    def test_skips_download_when_all_chunks_cached(self):
+        """Should not call snapshot_download when all needed chunks are cached."""
+        import os
+        import tempfile
+        from unittest.mock import MagicMock, patch
+
+        from tico.quantization.evaluation.lmms_eval_utils import (
+            _ensure_videomme_chunks_downloaded,
+        )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Create the cache with chunk 01 already present
+            repo_dir = os.path.join(tmpdir, "hub", "datasets--lmms-lab--Video-MME")
+            snap_dir = os.path.join(repo_dir, "snapshots", "abc123")
+            os.makedirs(snap_dir)
+            with open(os.path.join(snap_dir, "videos_chunked_01.zip"), "w") as f:
+                f.write("fake")
+
+            original_hf = os.environ.get("HF_HOME")
+            try:
+                os.environ["HF_HOME"] = tmpdir
+
+                with patch("huggingface_hub.snapshot_download") as mock_dl:
+                    _ensure_videomme_chunks_downloaded(limit=1)
+                    # Should NOT call snapshot_download since chunk 01 is cached
+                    mock_dl.assert_not_called()
+            finally:
+                if original_hf is not None:
+                    os.environ["HF_HOME"] = original_hf
+                else:
+                    os.environ.pop("HF_HOME", None)
+
+    def test_downloads_missing_chunks(self):
+        """Should download only missing chunks when some are cached."""
+        import os
+        import tempfile
+        from unittest.mock import patch
+
+        from tico.quantization.evaluation.lmms_eval_utils import (
+            _ensure_videomme_chunks_downloaded,
+        )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Create the cache with chunk 01 already present
+            repo_dir = os.path.join(tmpdir, "hub", "datasets--lmms-lab--Video-MME")
+            snap_dir = os.path.join(repo_dir, "snapshots", "abc123")
+            os.makedirs(snap_dir)
+            with open(os.path.join(snap_dir, "videos_chunked_01.zip"), "w") as f:
+                f.write("fake")
+
+            original_hf = os.environ.get("HF_HOME")
+            try:
+                os.environ["HF_HOME"] = tmpdir
+
+                with patch("huggingface_hub.snapshot_download") as mock_dl:
+                    # limit=31 needs 2 chunks (01 and 02), but 01 is cached
+                    _ensure_videomme_chunks_downloaded(limit=31)
+                    mock_dl.assert_called_once()
+                    call_kwargs = mock_dl.call_args
+                    allow_patterns = call_kwargs.kwargs.get(
+                        "allow_patterns"
+                    ) or call_kwargs[1].get("allow_patterns")
+                    # Should only download chunk 02 (01 is cached)
+                    self.assertIn("videos_chunked_02.zip", allow_patterns)
+                    self.assertNotIn("videos_chunked_01.zip", allow_patterns)
+            finally:
+                if original_hf is not None:
+                    os.environ["HF_HOME"] = original_hf
+                else:
+                    os.environ.pop("HF_HOME", None)
+
+
 class TestVerboseFlagPropagation(unittest.TestCase):
     """Tests for verbose flag propagation via LMMS_VERBOSE env var."""
 
@@ -386,20 +506,25 @@ def test_process_docs_filters_by_available_videos(self):
             finally:
                 vm_utils._data_dir = original_data_dir
 
-    def test_verbose_flag_controls_print(self):
-        """Print statements should be suppressed when LMMS_VERBOSE is not set."""
+    def test_is_verbose_reflects_runtime_env_changes(self):
+        """_is_verbose() should reflect runtime changes to LMMS_VERBOSE env var."""
         import os
 
-        from tico.quantization.evaluation.lmms_tasks.videomme_mini import (
-            utils as vm_utils,
+        from tico.quantization.evaluation.lmms_tasks.videomme_mini.utils import (
+            _is_verbose,
         )
 
         # Ensure verbose is off
         os.environ.pop("LMMS_VERBOSE", None)
-        # Re-evaluate _VERBOSE by reimporting the module is tricky,
-        # so we just test the _VERBOSE flag directly.
-        # Since _VERBOSE is evaluated at import time, we test the env var logic.
-        self.assertFalse(os.getenv("LMMS_VERBOSE", "").lower() in ("1", "true", "yes"))
+        self.assertFalse(_is_verbose())
+
+        # Set verbose on at runtime – _is_verbose() should pick it up immediately
+        os.environ["LMMS_VERBOSE"] = "1"
+        self.assertTrue(_is_verbose())
+
+        # Turn it off again
+        os.environ.pop("LMMS_VERBOSE", None)
+        self.assertFalse(_is_verbose())
 
 
 if __name__ == "__main__":