Stabilize Snapshot Tests by Normalizing Model Outputs

Author: rpancham

tests/model_serving/model_runtime/model_validation/utils.py

@@ -3,27 +3,45 @@
 from tests.model_serving.model_runtime.vllm.constant import VLLM_SUPPORTED_QUANTIZATION
 
 
+def extract_content_field(output: Any) -> str:
+    """
+    Extract the 'content' field from a typical VLLM chat response.
+    Returns an empty string if not found.
+    """
+    try:
+        return output["choices"][0]["message"].get("content", "").strip()
+    except (KeyError, IndexError, TypeError):
+        return ""
+
+
 def validate_supported_quantization_schema(q_type: str) -> None:
     if q_type not in VLLM_SUPPORTED_QUANTIZATION:
         raise ValueError(f"Unsupported quantization type: {q_type}")
 
 
-def validate_inference_output(*args: tuple[str, ...] | list[Any], response_snapshot: Any) -> None:
-    for data in args:
-        assert data == response_snapshot, f"output mismatch for {data}"
-
+def validate_inference_output(response_output: Any, expected_keywords: list[str]) -> None:
+    """
+    Validate inference response output using regex-based keyword checks.
 
-def safe_k8s_name(model_name: str, max_length: int = 20) -> str:
+    - Extracts 'content' field from model output.
+    - Fails if content is empty.
+    - Passes if any of the expected keywords/phrases are found in the content (case-insensitive).
     """
-    Create a safe Kubernetes name from model_name by truncating to max_length characters
-    and ensuring it follows Kubernetes naming conventions.
 
-    Args:
-        model_name: The original model name
-        max_length: Maximum length for the name (default: 20)
+    content = extract_content_field(response_output)
+    assert content, "Inference output is empty or missing 'content' field."
+    found_keywords = [kw for kw in expected_keywords if re.search(rf"\b{re.escape(kw)}\b", content, re.IGNORECASE)]
+    assert found_keywords, (
+        f"Expected one of {expected_keywords} in response content, "
+        f"but got: {content[:900]}..."  
+    )
+
+    print(f"Output validation passed. Found keywords: {found_keywords}")
+
 
-    Returns:
-        A valid Kubernetes name truncated to max_length characters
+def safe_k8s_name(model_name: str, max_length: int = 20) -> str:
+    """
+    Generate a Kubernetes-safe model name.
     """
     if not model_name:
         return "default-model"