Tests for hugging face scheduled sync

tests/model_registry/model_catalog/huggingface/conftest.py

@@ -1,7 +1,10 @@
 import pytest
+import time
 from huggingface_hub import HfApi
 from simple_logger.logger import get_logger
 
+from tests.model_registry.model_catalog.huggingface.utils import get_huggingface_model_from_api
+
 LOGGER = get_logger(name=__name__)
 
 
@@ -34,3 +37,31 @@ def num_models_from_hf_api_with_matching_criteria(request: pytest.FixtureRequest
         else:
             model_list.append(model.id)
     return len(model_list)
+
+
+@pytest.fixture(scope="module")
+def epoch_time_before_config_map_update() -> float:
+    """
+    Return the current epoch time in milliseconds when the test class starts.
+    Useful for comparing against timestamps created during test execution.
+    """
+    return float(time.time() * 1000)
+
+
+@pytest.fixture(scope="function")
+def initial_last_synced_values(
+    request: pytest.FixtureRequest,
+    model_catalog_rest_url: list[str],
+    model_registry_rest_headers: dict[str, str],
+) -> str:
+    """
+    Collect initial last_synced values for a given model.
+    """
+    result = get_huggingface_model_from_api(
+        model_registry_rest_headers=model_registry_rest_headers,
+        model_catalog_rest_url=model_catalog_rest_url,
+        model_name=request.param,
+        source_id="hf_id",
+    )
+
+    return result["customProperties"]["last_synced"]["string_value"]

tests/model_registry/model_catalog/huggingface/test_huggingface_model_validation.py

@@ -1,23 +1,68 @@
 import pytest
-from typing import Self
+from typing import Self, Generator
 from ocp_resources.config_map import ConfigMap
 from simple_logger.logger import get_logger
-from tests.model_registry.model_catalog.constants import HF_MODELS
+from tests.model_registry.model_catalog.constants import HF_MODELS, HF_SOURCE_ID
 from tests.model_registry.model_catalog.utils import (
     get_hf_catalog_str,
 )
 from tests.model_registry.model_catalog.huggingface.utils import (
     assert_huggingface_values_matches_model_catalog_api_values,
     wait_for_huggingface_retrival_match,
     wait_for_hugging_face_model_import,
+    wait_for_last_sync_update,
+    get_huggingface_model_from_api,
 )
+from huggingface_hub import HfApi
 from kubernetes.dynamic import DynamicClient
 
 LOGGER = get_logger(name=__name__)
 
-pytestmark = [
-    pytest.mark.usefixtures("updated_dsc_component_state_scope_session", "model_registry_namespace"),
-]
+
+class TestLastSyncedMetadataValidation:
+    """Test HuggingFace model last synced timestamp validation"""
+
+    @pytest.mark.parametrize(
+        "updated_catalog_config_map_scope_function, initial_last_synced_values, model_name",
+        [
+            pytest.param(
+                """
+catalogs:
+  - name: HuggingFace Hub
+    id: hf_id
+    type: hf
+    enabled: true
+    includedModels:
+    - microsoft/phi-2
+    properties:
+      syncInterval: '2m'
+""",
+                "microsoft/phi-2",
+                "microsoft/phi-2",
+                id="test_hf_last_synced_custom",
+            ),
+        ],
+        indirect=["updated_catalog_config_map_scope_function", "initial_last_synced_values"],
+    )
+    def test_huggingface_last_synced_custom(
+        self: Self,
+        updated_catalog_config_map_scope_function: Generator[ConfigMap, None, None],
+        initial_last_synced_values: str,
+        model_catalog_rest_url: list[str],
+        model_registry_rest_headers: dict[str, str],
+        model_name: str,
+    ):
+        """
+        Custom test for HuggingFace model last synced validation
+        """
+        # Get the model name from the parametrized test
+        wait_for_last_sync_update(
+            model_registry_rest_headers=model_registry_rest_headers,
+            model_catalog_rest_url=model_catalog_rest_url,
+            model_name=model_name,
+            source_id="hf_id",
+            initial_last_synced_values=float(initial_last_synced_values),
+        )
 
 
 @pytest.mark.parametrize(
@@ -34,10 +79,55 @@
     ],
     indirect=True,
 )
-@pytest.mark.usefixtures("updated_catalog_config_map")
+@pytest.mark.usefixtures("epoch_time_before_config_map_update", "updated_catalog_config_map")
 class TestHuggingFaceModelValidation:
     """Test HuggingFace model values by comparing values between HF API calls and Model Catalog api call"""
 
+    def test_huggingface_model_metadata_last_synced(
+        self: Self,
+        epoch_time_before_config_map_update: float,
+        model_catalog_rest_url: list[str],
+        model_registry_rest_headers: dict[str, str],
+        expected_catalog_values: dict[str, str],
+        huggingface_api: HfApi,
+    ):
+        """
+        Validate HuggingFace model last synced timestamp is properly updated
+        """
+        LOGGER.info(
+            f"Validating HuggingFace model last synced timestamps with {epoch_time_before_config_map_update} "
+            "epoch (milliseconds)"
+        )
+        error = {}
+        for model_name in expected_catalog_values:
+            result = get_huggingface_model_from_api(
+                model_catalog_rest_url=model_catalog_rest_url,
+                model_registry_rest_headers=model_registry_rest_headers,
+                model_name=model_name,
+                source_id=HF_SOURCE_ID,
+            )
+            error_msg = ""
+            if result["name"] != model_name:
+                error_msg += f"Expected model name {model_name}, but got {result['name']}. "
+
+            # Extract last_synced timestamp
+            last_synced = result["customProperties"]["last_synced"]["string_value"]
+            LOGGER.info(f"Model {model_name} last synced at: {last_synced}")
+
+            # Validate that last_synced field exists and is not empty
+            if not last_synced or last_synced == "":
+                error_msg += f"last_synced field is not present for model {model_name}. "
+            elif epoch_time_before_config_map_update > float(last_synced):
+                error_msg += (
+                    f"Model {model_name} last_synced ({last_synced}) should be after "
+                    f"test start time ({epoch_time_before_config_map_update}). "
+                )
+            if error_msg:
+                error[model_name] = error_msg
+        if error:
+            LOGGER.error(error)
+            pytest.fail("Last synced validation failed")
+
     def test_huggingface_model_metadata(
         self: Self,
         updated_catalog_config_map: tuple[ConfigMap, str, str],
@@ -127,7 +217,7 @@ def test_hugging_face_models(
         self: Self,
         admin_client: DynamicClient,
         model_registry_namespace: str,
-        updated_catalog_config_map_scope_function: ConfigMap,
+        updated_catalog_config_map_scope_function: Generator[ConfigMap, None, None],
         model_catalog_rest_url: list[str],
         model_registry_rest_headers: dict[str, str],
         huggingface_api: bool,

tests/model_registry/model_catalog/huggingface/utils.py

@@ -1,5 +1,6 @@
 import ast
 from typing import Any
+
 from simple_logger.logger import get_logger
 
 from tests.model_registry.model_catalog.constants import HF_SOURCE_ID
@@ -140,3 +141,54 @@ def wait_for_hugging_face_model_import(
     else:
         LOGGER.warning(f"No relevant log entry found: {log}")
         return False
+
+
+def get_huggingface_model_from_api(
+    model_catalog_rest_url: list[str],
+    model_registry_rest_headers: dict[str, str],
+    model_name: str,
+    source_id: str,
+) -> dict[str, Any]:
+    url = f"{model_catalog_rest_url[0]}sources/{source_id}/models/{model_name}"
+    return execute_get_command(
+        url=url,
+        headers=model_registry_rest_headers,
+    )
+
+
+@retry(wait_timeout=135, sleep=15)
+def wait_for_last_sync_update(
+    model_catalog_rest_url: list[str],
+    model_registry_rest_headers: dict[str, str],
+    model_name: str,
+    source_id: str,
+    initial_last_synced_values: float,
+) -> bool:
+    """Wait for the last_synced value to be updated with exact 120-second difference"""
+
+    result = get_huggingface_model_from_api(
+        model_registry_rest_headers=model_registry_rest_headers,
+        model_catalog_rest_url=model_catalog_rest_url,
+        model_name=model_name,
+        source_id=source_id,
+    )
+    current_last_synced = float(result["customProperties"]["last_synced"]["string_value"])
+    if current_last_synced != initial_last_synced_values:
+        # Calculate difference in milliseconds and convert to seconds
+        difference_seconds = int((current_last_synced - initial_last_synced_values) / 1000)
+
+        LOGGER.info(
+            f"Model {model_name}: initial={initial_last_synced_values}, current={current_last_synced}, "
+            f"diff={difference_seconds}s"
+        )
+        expected_diff = 120
+        if difference_seconds == expected_diff:
+            LOGGER.info(f"Model {model_name} successfully synced with correct interval ({difference_seconds}s)")
+            return True
+        else:
+            LOGGER.error(
+                f"Model {model_name}: sync interval should be {expected_diff}s, "
+                f"but found {difference_seconds}s (difference: {abs(difference_seconds - expected_diff)}s). "
+                f"Initial: {initial_last_synced_values}, Current: {current_last_synced}"
+            )
+    return False