feat: enhance vector store fixture with dataset support

Refactor the vector store fixture to support Dataset instances
for document uploads, improving test setup flexibility.
Update vector store upload functions to handle datasets and
their associated attributes. Remove deprecated constants and
adjust test cases to use the new dataset structure.

Signed-off-by: Jorge Garcia Oncins <jgarciao@redhat.com>
Made-with: Cursor

Author: jgarciao

tests/llama_stack/conftest.py

@@ -834,6 +834,11 @@ def vector_store(
         LOGGER.info(f"vector_store successfully created (provider_id={vector_io_provider}, id={vector_store.id})")
 
         if dataset or doc_sources:
+            if dataset and doc_sources:
+                LOGGER.warning(
+                    "Both 'dataset' and 'doc_sources' were provided; only 'dataset' will be used "
+                    "(these parameters are mutually exclusive)"
+                )
             try:
                 if dataset:
                     vector_store_upload_dataset(

tests/llama_stack/dataset/corpus/finance/documents.json

@@ -44,7 +44,7 @@
       "period_year": 2025,
       "document_type": "earnings_press_release",
       "language": "en",
-      "publication_date": 1738022400
+      "publication_date": 1769558400
     }
   }
 ]

tests/llama_stack/utils.py

@@ -79,7 +79,7 @@ def vector_store_create_and_poll(
     vector_store_id: str,
     file_id: str,
     *,
-    attributes: Any,
+    attributes: dict[str, str | int | float | bool] | None = None,
     poll_interval_sec: float = 5.0,
     wait_timeout: float = 240.0,
 ) -> VectorStoreFile:
@@ -290,7 +290,7 @@ def vector_store_create_file_from_path(
         _assert_file_uploaded(uploaded_file=uploaded_file, expected_purpose="assistants")
     LOGGER.info(f"Uploaded {file_path.name} (file_id={uploaded_file.id}) to the llama-stack files provider")
 
-    LOGGER.info(f"Adding uploaded file (filename{uploaded_file.filename} to vector store {vector_store.id}")
+    LOGGER.info(f"Adding uploaded file (filename={uploaded_file.filename}) to vector store {vector_store.id}")
     vs_file = vector_store_create_and_poll(
         llama_stack_client=llama_stack_client,
         vector_store_id=vector_store.id,
@@ -300,7 +300,7 @@ def vector_store_create_file_from_path(
     _assert_vector_store_file_attached(
         filename=uploaded_file.filename, vs_file=vs_file, vector_store_id=vector_store.id, attributes=attributes
     )
-    LOGGER.info(f"Added uploaded file (filename{uploaded_file.filename} to vector store {vector_store.id}")
+    LOGGER.info(f"Added uploaded file (filename={uploaded_file.filename}) to vector store {vector_store.id}")
     return vs_file
 
 

tests/llama_stack/vector_io/test_vector_stores.py

@@ -196,7 +196,8 @@ def test_response_file_search_tool_invocation(
         a completed file_search_call with results carrying file metadata, and that the
         message includes file_citation annotations with file_id and filename.
         """
-        vector_question = next(r.question for r in dataset.load_qa(retrieval_mode="vector"))
+        vector_question = next((r.question for r in dataset.load_qa(retrieval_mode="vector")), None)
+        assert vector_question is not None, "No vector-mode QA records found in dataset ground truth"
 
         response = unprivileged_llama_stack_client.responses.create(
             input=vector_question,

tests/model_explainability/evalhub/test_evalhub_health.py

@@ -1,22 +1,22 @@
 import pytest
 from ocp_resources.route import Route
 
-from tests.model_explainability.evalhub.utils import validate_evalhub_health
+from tests.model_explainability.evalhub.utils import validate_evalhub_health, validate_evalhub_providers
 
 
 @pytest.mark.parametrize(
     "model_namespace",
     [
         pytest.param(
-            {"name": "test-evalhub-health"},
+            {"name": "test-evalhub-health-providers"},
         ),
     ],
     indirect=True,
 )
 @pytest.mark.smoke
 @pytest.mark.model_explainability
-class TestEvalHubHealth:
-    """Tests for basic EvalHub service health and availability."""
+class TestEvalHub:
+    """Tests for basic EvalHub service health and providers."""
 
     def test_evalhub_health_endpoint(
         self,
@@ -30,3 +30,19 @@ def test_evalhub_health_endpoint(
             token=current_client_token,
             ca_bundle_file=evalhub_ca_bundle_file,
         )
+
+    def test_evalhub_providers_list(
+        self,
+        current_client_token: str,
+        evalhub_ca_bundle_file: str,
+        evalhub_route: Route,
+        model_namespace,
+    ) -> None:
+        """Test that the evaluations providers endpoint returns a non-empty list."""
+
+        validate_evalhub_providers(
+            host=evalhub_route.host,
+            token=current_client_token,
+            ca_bundle_file=evalhub_ca_bundle_file,
+            tenant=model_namespace.name,
+        )

tests/model_explainability/evalhub/utils.py

@@ -3,12 +3,31 @@
 from tests.model_explainability.evalhub.constants import (
     EVALHUB_HEALTH_PATH,
     EVALHUB_HEALTH_STATUS_HEALTHY,
+    EVALHUB_PROVIDERS_PATH,
 )
 from utilities.guardrails import get_auth_headers
 from utilities.opendatahub_logger import get_logger
 
 LOGGER = get_logger(name=__name__)
 
+TENANT_HEADER: str = "X-Tenant"
+
+
+def _build_headers(token: str, tenant: str | None = None) -> dict[str, str]:
+    """Build request headers with auth and optional tenant.
+
+    Args:
+        token: Bearer token for authentication.
+        tenant: Namespace for the X-Tenant header. Omitted if None.
+
+    Returns:
+        Headers dict.
+    """
+    headers = get_auth_headers(token=token)
+    if tenant is not None:
+        headers[TENANT_HEADER] = tenant
+    return headers
+
 
 def validate_evalhub_health(
     host: str,
@@ -45,3 +64,26 @@ def validate_evalhub_health(
         f"Expected status '{EVALHUB_HEALTH_STATUS_HEALTHY}', got '{data['status']}'"
     )
     assert "timestamp" in data, "Health response missing 'timestamp' field"
+
+
+def validate_evalhub_providers(
+    host: str,
+    token: str,
+    ca_bundle_file: str,
+    tenant: str | None = None,
+) -> dict:
+    """Smoke test for the EvalHub providers endpoint."""
+    url = f"https://{host}{EVALHUB_PROVIDERS_PATH}"
+
+    response = requests.get(
+        url=url,
+        headers=_build_headers(token=token, tenant=tenant),
+        verify=ca_bundle_file,
+        timeout=10,
+    )
+    response.raise_for_status()
+
+    data = response.json()
+    assert data.get("items"), f"Smoke test failed: Providers list is empty for tenant {tenant}"
+
+    return data