From a9886f3b835a91c53c28dc645d6c834a37a41ca3 Mon Sep 17 00:00:00 2001 From: Artemy Date: Tue, 16 Sep 2025 17:13:23 +0100 Subject: [PATCH 1/7] Add LlamaStack image validation tests and related fixture - Introduced a new fixture to retrieve the LlamaStack operator pod by label. - Added tests to verify that LlamaStack component images meet specified requirements, including image source and format. - Updated constants to include a label filter for the LlamaStack operator pod. --- tests/llama_stack/constants.py | 2 + .../llama_stack/image_validation/conftest.py | 20 +++++++ .../test_verify_lls_rhoai_images.py | 56 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 tests/llama_stack/image_validation/conftest.py create mode 100644 tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index 7b150ad7d..d23921eea 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -12,3 +12,5 @@ class Safety(str, Enum): class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" + +LLS_OPERATOR_POD_FILTER: str = "app.kubernetes.io/name=llama-stack-k8s-operator" diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py new file mode 100644 index 000000000..390e0c5e8 --- /dev/null +++ b/tests/llama_stack/image_validation/conftest.py @@ -0,0 +1,20 @@ +from typing import Generator, Any + +import pytest +from kubernetes.dynamic import DynamicClient +from pytest_testconfig import config as py_config + +from ocp_resources.pod import Pod +from tests.llama_stack.constants import LLS_OPERATOR_POD_FILTER +from utilities.general import wait_for_pods_by_labels + + +@pytest.fixture(scope="class") +def lls_operator_pod_by_label(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: + """Get the LLS operator pod.""" + yield wait_for_pods_by_labels( + admin_client=admin_client, + namespace=py_config["applications_namespace"], + label_selector=LLS_OPERATOR_POD_FILTER, + expected_num_pods=1, + )[0] diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py new file mode 100644 index 000000000..b2a694d3e --- /dev/null +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -0,0 +1,56 @@ +import pytest +from typing import Self, Set +from simple_logger.logger import get_logger +from kubernetes.dynamic import DynamicClient + +from utilities.general import ( + validate_container_images, +) +from ocp_resources.pod import Pod +from ocp_resources.namespace import Namespace + + +LOGGER = get_logger(name=__name__) + + +@pytest.mark.usefixtures( + "enabled_llama_stack_operator" +) + +@pytest.mark.parametrize( + "model_namespace", + [ + pytest.param( + {"name": "test-llamastack-image-validation"}, + ) + ], + indirect=True, +) + +@pytest.mark.rawdeployment +@pytest.mark.downstream_only +class TestLLSImages: + """ + Tests to verify that all LLS (LlamaStack) component images meet the requirements: + 1. Images are hosted in registry.redhat.io + 2. Images use sha256 digest instead of tags + 3. Images are listed in the CSV's relatedImages section + """ + + @pytest.mark.smoke + def test_verify_lls_operator_images( + self: Self, + admin_client: DynamicClient, + lls_operator_pod_by_label: Pod, + related_images_refs: Set[str], + ): + validation_errors = [] + for pod in [lls_operator_pod_by_label]: + validation_errors.extend( + validate_container_images( + pod=pod, valid_image_refs=related_images_refs, skip_patterns=["openshift-service-mesh"] + ) + ) + + if validation_errors: + pytest.fail("\n".join(validation_errors)) From e3f5a8bb3ee5c1e9d64ea65d0e1944b5457eac7e Mon Sep 17 00:00:00 2001 From: Artemy Date: Tue, 16 Sep 2025 17:13:23 +0100 Subject: [PATCH 2/7] Add LlamaStack image validation tests and related fixture - Introduced a new fixture to retrieve the LlamaStack operator pod by label. - Added tests to verify that LlamaStack component images meet specified requirements, including image source and format. - Updated constants to include a label filter for the LlamaStack operator pod. --- tests/llama_stack/constants.py | 2 + .../llama_stack/image_validation/conftest.py | 20 +++++++ .../test_verify_lls_rhoai_images.py | 56 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 tests/llama_stack/image_validation/conftest.py create mode 100644 tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index 7b150ad7d..d23921eea 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -12,3 +12,5 @@ class Safety(str, Enum): class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" + +LLS_OPERATOR_POD_FILTER: str = "app.kubernetes.io/name=llama-stack-k8s-operator" diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py new file mode 100644 index 000000000..390e0c5e8 --- /dev/null +++ b/tests/llama_stack/image_validation/conftest.py @@ -0,0 +1,20 @@ +from typing import Generator, Any + +import pytest +from kubernetes.dynamic import DynamicClient +from pytest_testconfig import config as py_config + +from ocp_resources.pod import Pod +from tests.llama_stack.constants import LLS_OPERATOR_POD_FILTER +from utilities.general import wait_for_pods_by_labels + + +@pytest.fixture(scope="class") +def lls_operator_pod_by_label(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: + """Get the LLS operator pod.""" + yield wait_for_pods_by_labels( + admin_client=admin_client, + namespace=py_config["applications_namespace"], + label_selector=LLS_OPERATOR_POD_FILTER, + expected_num_pods=1, + )[0] diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py new file mode 100644 index 000000000..b2a694d3e --- /dev/null +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -0,0 +1,56 @@ +import pytest +from typing import Self, Set +from simple_logger.logger import get_logger +from kubernetes.dynamic import DynamicClient + +from utilities.general import ( + validate_container_images, +) +from ocp_resources.pod import Pod +from ocp_resources.namespace import Namespace + + +LOGGER = get_logger(name=__name__) + + +@pytest.mark.usefixtures( + "enabled_llama_stack_operator" +) + +@pytest.mark.parametrize( + "model_namespace", + [ + pytest.param( + {"name": "test-llamastack-image-validation"}, + ) + ], + indirect=True, +) + +@pytest.mark.rawdeployment +@pytest.mark.downstream_only +class TestLLSImages: + """ + Tests to verify that all LLS (LlamaStack) component images meet the requirements: + 1. Images are hosted in registry.redhat.io + 2. Images use sha256 digest instead of tags + 3. Images are listed in the CSV's relatedImages section + """ + + @pytest.mark.smoke + def test_verify_lls_operator_images( + self: Self, + admin_client: DynamicClient, + lls_operator_pod_by_label: Pod, + related_images_refs: Set[str], + ): + validation_errors = [] + for pod in [lls_operator_pod_by_label]: + validation_errors.extend( + validate_container_images( + pod=pod, valid_image_refs=related_images_refs, skip_patterns=["openshift-service-mesh"] + ) + ) + + if validation_errors: + pytest.fail("\n".join(validation_errors)) From 70a19c92841c5f0d2ea5cdbcae5b70b7e226e76f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:21:32 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/llama_stack/constants.py | 1 + .../image_validation/test_verify_lls_rhoai_images.py | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index d23921eea..59da7174d 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -13,4 +13,5 @@ class Safety(str, Enum): class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" + LLS_OPERATOR_POD_FILTER: str = "app.kubernetes.io/name=llama-stack-k8s-operator" diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py index b2a694d3e..49f13dc47 100644 --- a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -7,16 +7,12 @@ validate_container_images, ) from ocp_resources.pod import Pod -from ocp_resources.namespace import Namespace LOGGER = get_logger(name=__name__) -@pytest.mark.usefixtures( - "enabled_llama_stack_operator" -) - +@pytest.mark.usefixtures("enabled_llama_stack_operator") @pytest.mark.parametrize( "model_namespace", [ @@ -26,7 +22,6 @@ ], indirect=True, ) - @pytest.mark.rawdeployment @pytest.mark.downstream_only class TestLLSImages: From 73557175848df06d1061b46e55802c115fc6361d Mon Sep 17 00:00:00 2001 From: Artemy Date: Tue, 23 Sep 2025 18:48:38 +0100 Subject: [PATCH 4/7] refactor: update LlamaStack test fixtures and constants - Renamed the LlamaStack distribution fixture for clarity. - Updated the pod label filter constant to reflect the core pod. - Modified the image validation test to use the new pod fixture name. --- tests/llama_stack/conftest.py | 2 +- tests/llama_stack/constants.py | 3 +-- tests/llama_stack/image_validation/conftest.py | 8 ++++---- .../image_validation/test_verify_lls_rhoai_images.py | 6 +++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/llama_stack/conftest.py b/tests/llama_stack/conftest.py index 7a6333989..7a8e02312 100644 --- a/tests/llama_stack/conftest.py +++ b/tests/llama_stack/conftest.py @@ -100,7 +100,7 @@ def llama_stack_distribution( ) -> Generator[LlamaStackDistribution, None, None]: with create_llama_stack_distribution( client=admin_client, - name="llama-stack-distribution", + name="test-lama-stack-distribution", namespace=model_namespace.name, replicas=1, server=llama_stack_server_config, diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index 59da7174d..5eb2b5335 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -13,5 +13,4 @@ class Safety(str, Enum): class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" - -LLS_OPERATOR_POD_FILTER: str = "app.kubernetes.io/name=llama-stack-k8s-operator" +LLS_CORE_POD_FILTER: str = "app=llama-stack" diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py index 390e0c5e8..bce7e426e 100644 --- a/tests/llama_stack/image_validation/conftest.py +++ b/tests/llama_stack/image_validation/conftest.py @@ -5,16 +5,16 @@ from pytest_testconfig import config as py_config from ocp_resources.pod import Pod -from tests.llama_stack.constants import LLS_OPERATOR_POD_FILTER +from tests.llama_stack.constants import LLS_CORE_POD_FILTER from utilities.general import wait_for_pods_by_labels @pytest.fixture(scope="class") -def lls_operator_pod_by_label(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: +def lls_pods(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: """Get the LLS operator pod.""" yield wait_for_pods_by_labels( admin_client=admin_client, - namespace=py_config["applications_namespace"], - label_selector=LLS_OPERATOR_POD_FILTER, + namespace=model_namespace.name, + label_selector=LLS_CORE_POD_FILTER, expected_num_pods=1, )[0] diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py index 49f13dc47..2ae2cc237 100644 --- a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -12,7 +12,7 @@ LOGGER = get_logger(name=__name__) -@pytest.mark.usefixtures("enabled_llama_stack_operator") +@pytest.mark.usefixtures("llama_stack_distribution") @pytest.mark.parametrize( "model_namespace", [ @@ -36,11 +36,11 @@ class TestLLSImages: def test_verify_lls_operator_images( self: Self, admin_client: DynamicClient, - lls_operator_pod_by_label: Pod, + lls_pods: Pod, related_images_refs: Set[str], ): validation_errors = [] - for pod in [lls_operator_pod_by_label]: + for pod in [lls_pods]: validation_errors.extend( validate_container_images( pod=pod, valid_image_refs=related_images_refs, skip_patterns=["openshift-service-mesh"] From 7e6e10e33143bfa90d16f5a1075b2886d957d602 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Sep 2025 18:25:38 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/llama_stack/constants.py | 2 ++ tests/llama_stack/image_validation/conftest.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/llama_stack/constants.py b/tests/llama_stack/constants.py index e3245ce23..5aae4deb1 100644 --- a/tests/llama_stack/constants.py +++ b/tests/llama_stack/constants.py @@ -15,8 +15,10 @@ class Safety(str, Enum): class Eval(str, Enum): TRUSTYAI_LMEVAL = "trustyai_lmeval" + LLS_CORE_POD_FILTER: str = "app=llama-stack" + @dataclass class TorchTuneTestExpectation: """Test expectation for TorchTune documentation questions.""" diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py index bce7e426e..f3256047f 100644 --- a/tests/llama_stack/image_validation/conftest.py +++ b/tests/llama_stack/image_validation/conftest.py @@ -2,7 +2,6 @@ import pytest from kubernetes.dynamic import DynamicClient -from pytest_testconfig import config as py_config from ocp_resources.pod import Pod from tests.llama_stack.constants import LLS_CORE_POD_FILTER From 71d2b223c69f27570bfb383f97d4cbdc6f575619 Mon Sep 17 00:00:00 2001 From: Artemy Date: Wed, 24 Sep 2025 12:45:09 +0100 Subject: [PATCH 6/7] refactor: fix review suggestions --- tests/llama_stack/image_validation/conftest.py | 2 +- .../test_verify_lls_rhoai_images.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/llama_stack/image_validation/conftest.py b/tests/llama_stack/image_validation/conftest.py index bce7e426e..335a2552e 100644 --- a/tests/llama_stack/image_validation/conftest.py +++ b/tests/llama_stack/image_validation/conftest.py @@ -11,7 +11,7 @@ @pytest.fixture(scope="class") def lls_pods(admin_client: DynamicClient, model_namespace) -> Generator[Pod, Any, Any]: - """Get the LLS operator pod.""" + """Get the LLS core deployment pod.""" yield wait_for_pods_by_labels( admin_client=admin_client, namespace=model_namespace.name, diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py index 2ae2cc237..706765d86 100644 --- a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -1,12 +1,11 @@ -import pytest from typing import Self, Set -from simple_logger.logger import get_logger -from kubernetes.dynamic import DynamicClient -from utilities.general import ( - validate_container_images, -) +from kubernetes.dynamic import DynamicClient from ocp_resources.pod import Pod +import pytest +from simple_logger.logger import get_logger + +from utilities.general import validate_container_images LOGGER = get_logger(name=__name__) @@ -26,7 +25,7 @@ @pytest.mark.downstream_only class TestLLSImages: """ - Tests to verify that all LLS (LlamaStack) component images meet the requirements: + Tests to verify that LLS (LlamaStack) Distribution image meets the requirements: 1. Images are hosted in registry.redhat.io 2. Images use sha256 digest instead of tags 3. Images are listed in the CSV's relatedImages section @@ -43,7 +42,7 @@ def test_verify_lls_operator_images( for pod in [lls_pods]: validation_errors.extend( validate_container_images( - pod=pod, valid_image_refs=related_images_refs, skip_patterns=["openshift-service-mesh"] + pod=pod, valid_image_refs=related_images_refs ) ) From 3b0284092944bd975937820f2133216ffd61f7fb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Sep 2025 13:46:31 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../image_validation/test_verify_lls_rhoai_images.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py index 706765d86..6f5f0324b 100644 --- a/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py +++ b/tests/llama_stack/image_validation/test_verify_lls_rhoai_images.py @@ -40,11 +40,7 @@ def test_verify_lls_operator_images( ): validation_errors = [] for pod in [lls_pods]: - validation_errors.extend( - validate_container_images( - pod=pod, valid_image_refs=related_images_refs - ) - ) + validation_errors.extend(validate_container_images(pod=pod, valid_image_refs=related_images_refs)) if validation_errors: pytest.fail("\n".join(validation_errors))