diff --git a/tests/llama_stack/conftest.py b/tests/llama_stack/conftest.py index 598114b1d..c3a74f978 100644 --- a/tests/llama_stack/conftest.py +++ b/tests/llama_stack/conftest.py @@ -107,7 +107,7 @@ def llama_stack_distribution( ) -> Generator[LlamaStackDistribution, None, None]: with create_llama_stack_distribution( client=admin_client, - name="llama-stack-distribution", + name="test-lama-stack-distribution", namespace=model_namespace.name, replicas=1, server=llama_stack_server_config, diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index 12eec4029..5aae4deb1 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -16,6 +16,9 @@ class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" +LLS_CORE_POD_FILTER: str = "app=llama-stack" + + @dataclass class TorchTuneTestExpectation: """Test expectation for TorchTune documentation questions.""" diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py new file mode 100644 index 000000000..79c98ee71 --- /dev/null +++ b/tests/llama_stack/image_validation/conftest.py @@ -0,0 +1,19 @@ +from typing import Generator, Any + +import pytest +from kubernetes.dynamic import DynamicClient + +from ocp_resources.pod import Pod +from tests.llama_stack.constants import LLS_CORE_POD_FILTER +from utilities.general import wait_for_pods_by_labels + + +@pytest.fixture(scope="class") +def lls_pods(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: + """Get the LLS core deployment pod.""" + yield wait_for_pods_by_labels( + admin_client=admin_client, + namespace=model_namespace.name, + label_selector=LLS_CORE_POD_FILTER, + expected_num_pods=1, + )[0] diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py new file mode 100644 index 000000000..6f5f0324b --- /dev/null +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -0,0 +1,46 @@ +from typing import Self, Set + +from kubernetes.dynamic import DynamicClient +from ocp_resources.pod import Pod +import pytest +from simple_logger.logger import get_logger + +from utilities.general import validate_container_images + + +LOGGER = get_logger(name=__name__) + + +@pytest.mark.usefixtures("llama_stack_distribution") +@pytest.mark.parametrize( + "model_namespace", + [ + pytest.param( + {"name": "test-llamastack-image-validation"}, + ) + ], + indirect=True, +) +@pytest.mark.rawdeployment +@pytest.mark.downstream_only +class TestLLSImages: + """ + Tests to verify that LLS (LlamaStack) Distribution image meets the requirements: + 1. Images are hosted in registry.redhat.io + 2. Images use sha256 digest instead of tags + 3. Images are listed in the CSV's relatedImages section + """ + + @pytest.mark.smoke + def test_verify_lls_operator_images( + self: Self, + admin_client: DynamicClient, + lls_pods: Pod, + related_images_refs: Set[str], + ): + validation_errors = [] + for pod in [lls_pods]: + validation_errors.extend(validate_container_images(pod=pod, valid_image_refs=related_images_refs)) + + if validation_errors: + pytest.fail("\n".join(validation_errors))