RHOAIENG-33693: Add LlamaStack image validation tests and related fixture (opendatahub-io#618)

* Add LlamaStack image validation tests and related fixture

- Introduced a new fixture to retrieve the LlamaStack operator pod by label.
- Added tests to verify that LlamaStack component images meet specified requirements, including image source and format.
- Updated constants to include a label filter for the LlamaStack operator pod.

* Add LlamaStack image validation tests and related fixture

- Introduced a new fixture to retrieve the LlamaStack operator pod by label.
- Added tests to verify that LlamaStack component images meet specified requirements, including image source and format.
- Updated constants to include a label filter for the LlamaStack operator pod.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* refactor: update LlamaStack test fixtures and constants

- Renamed the LlamaStack distribution fixture for clarity.
- Updated the pod label filter constant to reflect the core pod.
- Modified the image validation test to use the new pod fixture name.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* refactor: fix review suggestions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 2 people

tests/llama_stack/conftest.py

@@ -107,7 +107,7 @@ def llama_stack_distribution(
 ) -> Generator[LlamaStackDistribution, None, None]:
     with create_llama_stack_distribution(
         client=admin_client,
-        name="llama-stack-distribution",
+        name="test-lama-stack-distribution",
         namespace=model_namespace.name,
         replicas=1,
         server=llama_stack_server_config,

tests/llama_stack/constants.py

@@ -16,6 +16,9 @@ class Eval(str, Enum):
         TRUSTYAI_LMEVAL = "trustyai_lmeval"
 
 
+LLS_CORE_POD_FILTER: str = "app=llama-stack"
+
+
 @dataclass
 class TorchTuneTestExpectation:
     """Test expectation for TorchTune documentation questions."""

tests/llama_stack/image_validation/conftest.py

@@ -0,0 +1,19 @@
+from typing import Generator, Any
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+
+from ocp_resources.pod import Pod
+from tests.llama_stack.constants import LLS_CORE_POD_FILTER
+from utilities.general import wait_for_pods_by_labels
+
+
+@pytest.fixture(scope="class")
+def lls_pods(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]:
+    """Get the LLS core deployment pod."""
+    yield wait_for_pods_by_labels(
+        admin_client=admin_client,
+        namespace=model_namespace.name,
+        label_selector=LLS_CORE_POD_FILTER,
+        expected_num_pods=1,
+    )[0]

tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py

@@ -0,0 +1,46 @@
+from typing import Self, Set
+
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.pod import Pod
+import pytest
+from simple_logger.logger import get_logger
+
+from utilities.general import validate_container_images
+
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.usefixtures("llama_stack_distribution")
+@pytest.mark.parametrize(
+    "model_namespace",
+    [
+        pytest.param(
+            {"name": "test-llamastack-image-validation"},
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.rawdeployment
+@pytest.mark.downstream_only
+class TestLLSImages:
+    """
+    Tests to verify that LLS (LlamaStack) Distribution image meets the requirements:
+    1. Images are hosted in registry.redhat.io
+    2. Images use sha256 digest instead of tags
+    3. Images are listed in the CSV's relatedImages section
+    """
+
+    @pytest.mark.smoke
+    def test_verify_lls_operator_images(
+        self: Self,
+        admin_client: DynamicClient,
+        lls_pods: Pod,
+        related_images_refs: Set[str],
+    ):
+        validation_errors = []
+        for pod in [lls_pods]:
+            validation_errors.extend(validate_container_images(pod=pod, valid_image_refs=related_images_refs))
+
+        if validation_errors:
+            pytest.fail("\n".join(validation_errors))