diff --git a/tests/conftest.py b/tests/conftest.py index 10c7381cd..98a585a72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -531,13 +531,29 @@ def cluster_monitoring_config( @pytest.fixture(scope="class") def unprivileged_model_namespace( - request: FixtureRequest, admin_client: DynamicClient, unprivileged_client: DynamicClient + request: FixtureRequest, + pytestconfig: pytest.Config, + admin_client: DynamicClient, + unprivileged_client: DynamicClient, + teardown_resources: bool, ) -> Generator[Namespace, Any, Any]: if request.param.get("modelmesh-enabled"): request.getfixturevalue(argname="enabled_modelmesh_in_dsc") - with create_ns(admin_client=admin_client, unprivileged_client=unprivileged_client, pytest_request=request) as ns: + ns = Namespace(client=unprivileged_client, name=request.param["name"]) + if pytestconfig.option.post_upgrade: yield ns + ns.client = admin_client + if teardown_resources: + ns.clean_up() + else: + with create_ns( + admin_client=admin_client, + unprivileged_client=unprivileged_client, + pytest_request=request, + teardown=teardown_resources, + ) as ns: + yield ns # MinIo diff --git a/tests/fixtures/vector_io.py b/tests/fixtures/vector_io.py index 01678c6d9..96f8c3c37 100644 --- a/tests/fixtures/vector_io.py +++ b/tests/fixtures/vector_io.py @@ -152,11 +152,13 @@ def _factory(provider_name: str) -> list[Dict[str, Any]]: @pytest.fixture(scope="class") def vector_io_secret( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Secret, Any, Any]: """Create a secret for the vector I/O providers""" - with Secret( + secret = Secret( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-secret", @@ -167,17 +169,26 @@ def vector_io_secret( "pgvector-password": PGVECTOR_PASSWORD, "milvus-token": MILVUS_TOKEN, }, - ) as secret: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield secret + secret.clean_up() + else: + with secret: + yield secret @pytest.fixture(scope="class") def etcd_deployment( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy an etcd instance for vector I/O provider testing.""" - with Deployment( + deployment = Deployment( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-etcd-deployment", @@ -185,19 +196,28 @@ def etcd_deployment( selector={"matchLabels": {"app": "etcd"}}, strategy={"type": "Recreate"}, template=get_etcd_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=120) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=120) + yield deployment @pytest.fixture(scope="class") def etcd_service( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the etcd deployment.""" - with Service( + service = Service( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-etcd-service", @@ -209,24 +229,33 @@ def etcd_service( ], selector={"app": "etcd"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service @pytest.fixture(scope="class") def remote_milvus_deployment( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, etcd_deployment: Deployment, etcd_service: Service, vector_io_secret: Secret, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy a remote Milvus instance for vector I/O provider testing.""" _ = etcd_deployment _ = etcd_service _ = vector_io_secret - with Deployment( + deployment = Deployment( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-milvus-deployment", @@ -235,22 +264,31 @@ def remote_milvus_deployment( selector={"matchLabels": {"app": "milvus-standalone"}}, strategy={"type": "Recreate"}, template=get_milvus_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=240) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=240) + yield deployment @pytest.fixture(scope="class") def milvus_service( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, remote_milvus_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the remote Milvus deployment.""" _ = remote_milvus_deployment - with Service( + service = Service( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-milvus-service", @@ -263,8 +301,15 @@ def milvus_service( ], selector={"app": "milvus-standalone"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service def get_milvus_deployment_template() -> Dict[str, Any]: @@ -344,14 +389,16 @@ def get_etcd_deployment_template() -> Dict[str, Any]: @pytest.fixture(scope="class") def pgvector_deployment( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, vector_io_secret: Secret, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy a PGVector instance for vector I/O provider testing.""" _ = vector_io_secret - with Deployment( + deployment = Deployment( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-pgvector-deployment", @@ -360,22 +407,31 @@ def pgvector_deployment( selector={"matchLabels": {"app": "pgvector"}}, strategy={"type": "Recreate"}, template=get_pgvector_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=240) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=240) + yield deployment @pytest.fixture(scope="class") def pgvector_service( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, pgvector_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the PGVector deployment.""" _ = pgvector_deployment - with Service( + service = Service( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-pgvector-service", @@ -388,8 +444,15 @@ def pgvector_service( ], selector={"app": "pgvector"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service def get_pgvector_deployment_template() -> Dict[str, Any]: @@ -439,14 +502,16 @@ def get_pgvector_deployment_template() -> Dict[str, Any]: @pytest.fixture(scope="class") def qdrant_deployment( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, vector_io_secret: Secret, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy a Qdrant instance for vector I/O provider testing.""" _ = vector_io_secret - with Deployment( + deployment = Deployment( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-qdrant-deployment", @@ -455,22 +520,31 @@ def qdrant_deployment( selector={"matchLabels": {"app": "qdrant"}}, strategy={"type": "Recreate"}, template=get_qdrant_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=240) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=240) + yield deployment @pytest.fixture(scope="class") def qdrant_service( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, qdrant_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the Qdrant deployment.""" _ = qdrant_deployment - with Service( + service = Service( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-qdrant-service", @@ -488,8 +562,15 @@ def qdrant_service( ], selector={"app": "qdrant"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service def get_qdrant_deployment_template() -> Dict[str, Any]: diff --git a/tests/llama_stack/conftest.py b/tests/llama_stack/conftest.py index 65ce07a9e..d6fe0edfc 100644 --- a/tests/llama_stack/conftest.py +++ b/tests/llama_stack/conftest.py @@ -66,7 +66,14 @@ IBM_EARNINGS_DOC_URL = "https://www.ibm.com/downloads/documents/us-en/1550f7eea8c0ded6" -distribution_name = generate_random_name(prefix="llama-stack-distribution") +UPGRADE_DISTRIBUTION_NAME = "llama-stack-distribution-upgrade" + + +@pytest.fixture(scope="class") +def distribution_name(pytestconfig: pytest.Config) -> str: + if pytestconfig.option.pre_upgrade or pytestconfig.option.post_upgrade: + return UPGRADE_DISTRIBUTION_NAME + return generate_random_name(prefix="llama-stack-distribution") @pytest.fixture(scope="class") @@ -84,9 +91,11 @@ def enabled_llama_stack_operator(dsc_resource: DataScienceCluster) -> Generator[ @pytest.fixture(scope="class") def llama_stack_server_config( request: FixtureRequest, - vector_io_provider_deployment_config_factory: Callable[[str], list[Dict[str, str]]], - files_provider_config_factory: Callable[[str], list[Dict[str, str]]], -) -> Dict[str, Any]: + pytestconfig: pytest.Config, + distribution_name: str, + vector_io_provider_deployment_config_factory: Callable[[str], list[dict[str, str]]], + files_provider_config_factory: Callable[[str], list[dict[str, str]]], +) -> dict[str, Any]: """ Generate server configuration for LlamaStack distribution deployment and deploy vector I/O provider resources. @@ -293,53 +302,89 @@ def test_with_remote_milvus(llama_stack_server_config): @pytest.fixture(scope="class") def llama_stack_distribution_secret( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Secret, Any, Any]: - with Secret( + secret = Secret( client=admin_client, namespace=model_namespace.name, name="llamastack-distribution-secret", type="Opaque", string_data=LLAMA_STACK_DISTRIBUTION_SECRET_DATA, - ) as secret: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield secret + secret.clean_up() + else: + with secret: + yield secret @pytest.fixture(scope="class") def unprivileged_llama_stack_distribution_secret( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Secret, Any, Any]: - with Secret( + secret = Secret( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="llamastack-distribution-secret", type="Opaque", string_data=LLAMA_STACK_DISTRIBUTION_SECRET_DATA, - ) as secret: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield secret + secret.clean_up() + else: + with secret: + yield secret @pytest.fixture(scope="class") def unprivileged_llama_stack_distribution( + pytestconfig: pytest.Config, + distribution_name: str, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, enabled_llama_stack_operator: DataScienceCluster, request: FixtureRequest, llama_stack_server_config: dict[str, Any], + ci_s3_bucket_name: str, + ci_s3_bucket_endpoint: str, + ci_s3_bucket_region: str, + aws_access_key_id: str, + aws_secret_access_key: str, + teardown_resources: bool, unprivileged_llama_stack_distribution_secret: Secret, unprivileged_postgres_deployment: Deployment, unprivileged_postgres_service: Service, -) -> Generator[LlamaStackDistribution, None, None]: - # Distribution name needs a random substring due to bug RHAIENG-999 / RHAIENG-1139 - distribution_name = generate_random_name(prefix="llama-stack-distribution") +) -> Generator[LlamaStackDistribution, Any, Any]: + if pytestconfig.option.post_upgrade: + lls_dist = LlamaStackDistribution( + client=unprivileged_client, + name=distribution_name, + namespace=unprivileged_model_namespace.name, + ensure_exists=True, + ) + lls_dist.wait_for_status(status=LlamaStackDistribution.Status.READY, timeout=600) + yield lls_dist + lls_dist.clean_up() + return with create_llama_stack_distribution( client=unprivileged_client, name=distribution_name, namespace=unprivileged_model_namespace.name, replicas=1, server=llama_stack_server_config, + teardown=teardown_resources, ) as lls_dist: lls_dist.wait_for_status(status=LlamaStackDistribution.Status.READY, timeout=600) yield lls_dist @@ -347,22 +392,41 @@ def unprivileged_llama_stack_distribution( @pytest.fixture(scope="class") def llama_stack_distribution( + pytestconfig: pytest.Config, + distribution_name: str, admin_client: DynamicClient, model_namespace: Namespace, enabled_llama_stack_operator: DataScienceCluster, request: FixtureRequest, llama_stack_server_config: dict[str, Any], + ci_s3_bucket_name: str, + ci_s3_bucket_endpoint: str, + ci_s3_bucket_region: str, + aws_access_key_id: str, + aws_secret_access_key: str, + teardown_resources: bool, llama_stack_distribution_secret: Secret, postgres_deployment: Deployment, postgres_service: Service, -) -> Generator[LlamaStackDistribution, None, None]: - # Distribution name needs a random substring due to bug RHAIENG-999 / RHAIENG-1139 +) -> Generator[LlamaStackDistribution, Any, Any]: + if pytestconfig.option.post_upgrade: + lls_dist = LlamaStackDistribution( + client=admin_client, + name=distribution_name, + namespace=model_namespace.name, + ensure_exists=True, + ) + lls_dist.wait_for_status(status=LlamaStackDistribution.Status.READY, timeout=600) + yield lls_dist + lls_dist.clean_up() + return with create_llama_stack_distribution( client=admin_client, name=distribution_name, namespace=model_namespace.name, replicas=1, server=llama_stack_server_config, + teardown=teardown_resources, ) as lls_dist: lls_dist.wait_for_status(status=LlamaStackDistribution.Status.READY, timeout=600) yield lls_dist @@ -454,9 +518,11 @@ def llama_stack_distribution_deployment( def _create_llama_stack_test_route( + pytestconfig: pytest.Config, client: DynamicClient, namespace: Namespace, deployment: Deployment, + teardown_resources: bool, ) -> Generator[Route, Any, Any]: """ Creates a Route for LlamaStack distribution with TLS configuration. @@ -469,56 +535,107 @@ def _create_llama_stack_test_route( Yields: Generator[Route, Any, Any]: Route resource with TLS edge termination """ - route_name = generate_random_name(prefix="llama-stack", length=12) + if pytestconfig.option.pre_upgrade or pytestconfig.option.post_upgrade: + # Keep the upgrade route name short to avoid OpenShift-generated host labels + # exceeding the DNS label limit (63 chars). + route_name = "lls-upg-route" + upgrade_route_patch = { + "spec": { + "tls": { + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect", + } + }, + "metadata": { + "annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: "10m"}, + }, + } + else: + route_name = generate_random_name(prefix="llama-stack", length=12) + + if pytestconfig.option.post_upgrade: + route = Route( + client=client, + namespace=namespace.name, + name=route_name, + ensure_exists=True, + ) + ResourceEditor( + patches={ + route: upgrade_route_patch, + } + ).update() + route.wait(timeout=60) + yield route + if teardown_resources: + route.clean_up() + return + with Route( client=client, namespace=namespace.name, name=route_name, service=f"{deployment.name}-service", wait_for_resource=True, + teardown=teardown_resources, ) as route: - with ResourceEditor( - patches={ - route: { - "spec": { - "tls": { - "termination": "edge", - "insecureEdgeTerminationPolicy": "Redirect", - } - }, - "metadata": { - "annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: "10m"}, - }, + if pytestconfig.option.pre_upgrade: + ResourceEditor( + patches={ + route: upgrade_route_patch, } - } - ): - route.wait(timeout=60) - yield route + ).update() + else: + ResourceEditor( + patches={ + route: { + "spec": { + "tls": { + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect", + } + }, + "metadata": { + "annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: "10m"}, + }, + } + } + ).update() + route.wait(timeout=60) + yield route @pytest.fixture(scope="class") def unprivileged_llama_stack_test_route( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, unprivileged_llama_stack_distribution_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Route, Any, Any]: yield from _create_llama_stack_test_route( + pytestconfig=pytestconfig, client=unprivileged_client, namespace=unprivileged_model_namespace, deployment=unprivileged_llama_stack_distribution_deployment, + teardown_resources=teardown_resources, ) @pytest.fixture(scope="class") def llama_stack_test_route( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, llama_stack_distribution_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Route, Any, Any]: yield from _create_llama_stack_test_route( + pytestconfig=pytestconfig, client=admin_client, namespace=model_namespace, deployment=llama_stack_distribution_deployment, + teardown_resources=teardown_resources, ) @@ -659,7 +776,9 @@ def vector_store( unprivileged_llama_stack_client: LlamaStackClient, llama_stack_models: ModelInfo, request: FixtureRequest, -) -> Generator[VectorStore, None, None]: + pytestconfig: pytest.Config, + teardown_resources: bool, +) -> Generator[VectorStore, Any, Any]: """ Creates a vector store for testing and automatically cleans it up. @@ -677,29 +796,45 @@ def vector_store( params = getattr(request, "param", {"vector_io_provider": "milvus"}) vector_io_provider = str(params.get("vector_io_provider")) - vector_store = unprivileged_llama_stack_client.vector_stores.create( - name="test_vector_store", - extra_body={ - "embedding_model": llama_stack_models.embedding_model.id, - "embedding_dimension": llama_stack_models.embedding_dimension, - "provider_id": vector_io_provider, - }, - ) - LOGGER.info(f"vector_store successfully created (provider_id={vector_io_provider}, id={vector_store.id})") + if pytestconfig.option.post_upgrade: + vector_store = next( + ( + vs + for vs in unprivileged_llama_stack_client.vector_stores.list().data + if getattr(vs, "name", "") == "test_vector_store" + ), + None, + ) + if not vector_store: + raise ValueError("Expected vector store 'test_vector_store' to exist in post-upgrade run") + LOGGER.info(f"Reusing existing vector_store in post-upgrade run (id={vector_store.id})") + else: + vector_store = unprivileged_llama_stack_client.vector_stores.create( + name="test_vector_store", + extra_body={ + "embedding_model": llama_stack_models.embedding_model.id, + "embedding_dimension": llama_stack_models.embedding_dimension, + "provider_id": vector_io_provider, + }, + ) + LOGGER.info(f"vector_store successfully created (provider_id={vector_io_provider}, id={vector_store.id})") yield vector_store - try: - unprivileged_llama_stack_client.vector_stores.delete(vector_store_id=vector_store.id) - LOGGER.info(f"Deleted vector store {vector_store.id}") - except Exception as e: - LOGGER.warning(f"Failed to delete vector store {vector_store.id}: {e}") + if teardown_resources: + try: + unprivileged_llama_stack_client.vector_stores.delete(vector_store_id=vector_store.id) + LOGGER.info(f"Deleted vector store {vector_store.id}") + except Exception as e: # noqa: BLE001 + LOGGER.warning(f"Failed to delete vector store {vector_store.id}: {e}") @pytest.fixture(scope="class") def vector_store_with_example_docs( - unprivileged_llama_stack_client: LlamaStackClient, vector_store: VectorStore -) -> Generator[VectorStore, None, None]: + unprivileged_llama_stack_client: LlamaStackClient, + vector_store: VectorStore, + pytestconfig: pytest.Config, +) -> Generator[VectorStore, Any, Any]: """ Creates a vector store with the IBM fourth-quarter 2025 earnings report uploaded. @@ -714,23 +849,28 @@ def vector_store_with_example_docs( Yields: Vector store object with uploaded IBM earnings report document """ - vector_store_create_file_from_url( - url=IBM_EARNINGS_DOC_URL, - llama_stack_client=unprivileged_llama_stack_client, - vector_store=vector_store, - ) + if pytestconfig.option.post_upgrade: + LOGGER.info("Post-upgrade run: reusing vector store docs without uploading new files") + else: + vector_store_create_file_from_url( + url=IBM_EARNINGS_DOC_URL, + llama_stack_client=unprivileged_llama_stack_client, + vector_store=vector_store, + ) yield vector_store @pytest.fixture(scope="class") def unprivileged_postgres_service( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, unprivileged_postgres_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the unprivileged postgres deployment.""" - with Service( + service = Service( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-postgres-service", @@ -742,17 +882,26 @@ def unprivileged_postgres_service( ], selector={"app": "postgres"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service @pytest.fixture(scope="class") def unprivileged_postgres_deployment( + pytestconfig: pytest.Config, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy a Postgres instance for vector I/O provider testing with unprivileged client.""" - with Deployment( + deployment = Deployment( client=unprivileged_client, namespace=unprivileged_model_namespace.name, name="vector-io-postgres-deployment", @@ -761,20 +910,29 @@ def unprivileged_postgres_deployment( selector={"matchLabels": {"app": "postgres"}}, strategy={"type": "Recreate"}, template=get_postgres_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=240) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=240) + yield deployment @pytest.fixture(scope="class") def postgres_service( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, postgres_deployment: Deployment, + teardown_resources: bool, ) -> Generator[Service, Any, Any]: """Create a service for the postgres deployment.""" - with Service( + service = Service( client=admin_client, namespace=model_namespace.name, name="vector-io-postgres-service", @@ -786,17 +944,26 @@ def postgres_service( ], selector={"app": "postgres"}, wait_for_resource=True, - ) as service: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield service + service.clean_up() + else: + with service: + yield service @pytest.fixture(scope="class") def postgres_deployment( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, + teardown_resources: bool, ) -> Generator[Deployment, Any, Any]: """Deploy a Postgres instance for vector I/O provider testing.""" - with Deployment( + deployment = Deployment( client=admin_client, namespace=model_namespace.name, name="vector-io-postgres-deployment", @@ -805,10 +972,17 @@ def postgres_deployment( selector={"matchLabels": {"app": "postgres"}}, strategy={"type": "Recreate"}, template=get_postgres_deployment_template(), - teardown=True, - ) as deployment: + teardown=teardown_resources, + ensure_exists=pytestconfig.option.post_upgrade, + ) + if pytestconfig.option.post_upgrade: deployment.wait_for_replicas(deployed=True, timeout=240) yield deployment + deployment.clean_up() + else: + with deployment: + deployment.wait_for_replicas(deployed=True, timeout=240) + yield deployment def get_postgres_deployment_template() -> Dict[str, Any]: diff --git a/tests/llama_stack/eval/conftest.py b/tests/llama_stack/eval/conftest.py index cd96134fb..ab6aaad6c 100644 --- a/tests/llama_stack/eval/conftest.py +++ b/tests/llama_stack/eval/conftest.py @@ -17,59 +17,81 @@ @pytest.fixture(scope="class") -def dataset_pvc(admin_client, model_namespace) -> Generator[PersistentVolumeClaim, Any, Any]: +def dataset_pvc( + pytestconfig: pytest.Config, + admin_client: DynamicClient, + model_namespace: Namespace, + teardown_resources: bool, +) -> Generator[PersistentVolumeClaim, Any, Any]: """ Creates a PVC to store the custom dataset. """ - with PersistentVolumeClaim( + pvc = PersistentVolumeClaim( client=admin_client, namespace=model_namespace.name, name="dataset-pvc", size="1Gi", accessmodes="ReadWriteOnce", label={"app.kubernetes.io/name": "dataset-storage"}, - ) as pvc: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield pvc + pvc.clean_up() + else: + with pvc: + yield pvc @pytest.fixture(scope="class") def dataset_upload( - admin_client: DynamicClient, model_namespace: Namespace, dataset_pvc: PersistentVolumeClaim -) -> Generator[dict[str, Any], None, None]: + pytestconfig: pytest.Config, + admin_client: DynamicClient, + model_namespace: Namespace, + dataset_pvc: PersistentVolumeClaim, +) -> Generator[dict[str, Any], Any, Any]: """ Copies dataset files from an image into the PVC at the location expected by LM-Eval """ dataset_target_path = "/opt/app-root/src/hf_home" - with Pod( - client=admin_client, - namespace=model_namespace.name, - name="dataset-copy-to-pvc", - label={"trustyai-tests": "dataset-upload"}, - security_context={"fsGroup": 1001, "seccompProfile": {"type": "RuntimeDefault"}}, - containers=[ - { - "name": "dataset-copy-to-pvc", - "image": DK_CUSTOM_DATASET_IMAGE, - "command": ["/bin/sh", "-c", "cp --verbose -r /models/* /mnt/pvc"], - "securityContext": { - "runAsUser": 1001, - "runAsNonRoot": True, - "allowPrivilegeEscalation": False, - "capabilities": {"drop": ["ALL"]}, - }, - "volumeMounts": [{"mountPath": "/mnt/pvc", "name": "pvc-volume"}], - } - ], - restart_policy="Never", - volumes=[{"name": "pvc-volume", "persistentVolumeClaim": {"claimName": dataset_pvc.name}}], - ) as pod: - pod.wait_for_status(status=Pod.Status.SUCCEEDED) + if pytestconfig.option.post_upgrade: + # In post-upgrade runs, only reuse expected dataset location from pre-upgrade. yield { - "pod": pod, + "pod": None, "dataset_path": f"{dataset_target_path}/example-dk-bench-input-bmo.jsonl", } + else: + with Pod( + client=admin_client, + namespace=model_namespace.name, + name="dataset-copy-to-pvc", + label={"trustyai-tests": "dataset-upload"}, + security_context={"fsGroup": 1001, "seccompProfile": {"type": "RuntimeDefault"}}, + containers=[ + { + "name": "dataset-copy-to-pvc", + "image": DK_CUSTOM_DATASET_IMAGE, + "command": ["/bin/sh", "-c", "cp --verbose -r /models/* /mnt/pvc"], + "securityContext": { + "runAsUser": 1001, + "runAsNonRoot": True, + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + }, + "volumeMounts": [{"mountPath": "/mnt/pvc", "name": "pvc-volume"}], + } + ], + restart_policy="Never", + volumes=[{"name": "pvc-volume", "persistentVolumeClaim": {"claimName": dataset_pvc.name}}], + ) as pod: + pod.wait_for_status(status=Pod.Status.SUCCEEDED) + yield { + "pod": pod, + "dataset_path": f"{dataset_target_path}/example-dk-bench-input-bmo.jsonl", + } @pytest.fixture(scope="function") @@ -98,17 +120,19 @@ def teardown_lmeval_job_pod(admin_client, model_namespace) -> None: @pytest.fixture(scope="class") def dspa( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, minio_pod: Pod, minio_service: Service, dspa_s3_secret: Secret, + teardown_resources: bool, ) -> Generator[DataSciencePipelinesApplication, Any, Any]: """ Creates a DataSciencePipelinesApplication with MinIO object storage. """ - with DataSciencePipelinesApplication( + dspa_resource = DataSciencePipelinesApplication( client=admin_client, name="dspa", namespace=model_namespace.name, @@ -154,25 +178,39 @@ def dspa( "deploy": True, "cronScheduleTimezone": "UTC", }, - ) as dspa_resource: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: wait_for_dspa_pods( admin_client=admin_client, namespace=model_namespace.name, dspa_name=dspa_resource.name, ) yield dspa_resource + dspa_resource.clean_up() + else: + with dspa_resource: + wait_for_dspa_pods( + admin_client=admin_client, + namespace=model_namespace.name, + dspa_name=dspa_resource.name, + ) + yield dspa_resource @pytest.fixture(scope="class") def dspa_s3_secret( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, minio_service: Service, + teardown_resources: bool, ) -> Generator[Secret, Any, Any]: """ Creates a secret for DSPA S3 credentials using MinIO. """ - with Secret( + secret = Secret( client=admin_client, name="dashboard-dspa-secret", namespace=model_namespace.name, @@ -181,8 +219,15 @@ def dspa_s3_secret( "AWS_SECRET_ACCESS_KEY": MinIo.Credentials.SECRET_KEY_VALUE, "AWS_DEFAULT_REGION": "us-east-1", }, - ) as secret: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade: yield secret + secret.clean_up() + else: + with secret: + yield secret @pytest.fixture(scope="class") diff --git a/tests/llama_stack/inference/upgrade/test_upgrade_chat_completions.py b/tests/llama_stack/inference/upgrade/test_upgrade_chat_completions.py new file mode 100644 index 000000000..2040ab423 --- /dev/null +++ b/tests/llama_stack/inference/upgrade/test_upgrade_chat_completions.py @@ -0,0 +1,81 @@ +import pytest +from llama_stack_client import LlamaStackClient + +from tests.llama_stack.constants import ModelInfo + + +def _assert_chat_completion_ack( + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, +) -> None: + response = unprivileged_llama_stack_client.chat.completions.create( + model=llama_stack_models.model_id, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Just respond ACK."}, + ], + temperature=0, + ) + assert len(response.choices) > 0, "No response after basic inference on llama-stack server" + + content = response.choices[0].message.content + assert content is not None, "LLM response content is None" + assert "ack" in content.lower(), "The LLM did not provide the expected answer to the prompt" + + +@pytest.mark.parametrize( + "unprivileged_model_namespace", + [ + pytest.param( + {"name": "test-llamastack-infer-chat-upgrade"}, + ), + ], + indirect=True, +) +@pytest.mark.llama_stack +class TestPreUpgradeLlamaStackInferenceCompletions: + @pytest.mark.pre_upgrade + def test_inference_chat_completion_pre_upgrade( + self, + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, + ) -> None: + """Verify chat completion returns ACK before upgrade. + + Given: A running unprivileged LlamaStack distribution. + When: A deterministic chat completion request is sent. + Then: The response contains at least one choice with non-empty ACK content. + """ + _assert_chat_completion_ack( + unprivileged_llama_stack_client=unprivileged_llama_stack_client, + llama_stack_models=llama_stack_models, + ) + + +@pytest.mark.parametrize( + "unprivileged_model_namespace", + [ + pytest.param( + {"name": "test-llamastack-infer-chat-upgrade"}, + ), + ], + indirect=True, +) +@pytest.mark.llama_stack +class TestPostUpgradeLlamaStackInferenceCompletions: + @pytest.mark.post_upgrade + def test_inference_chat_completion_post_upgrade( + self, + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, + ) -> None: + """Verify chat completion returns ACK after upgrade. + + Given: A pre-existing unprivileged LlamaStack distribution after platform upgrade. + When: A deterministic chat completion request is sent. + Then: The response contains at least one choice with non-empty ACK content. + """ + _assert_chat_completion_ack( + unprivileged_llama_stack_client=unprivileged_llama_stack_client, + llama_stack_models=llama_stack_models, + ) diff --git a/tests/llama_stack/safety/conftest.py b/tests/llama_stack/safety/conftest.py index 0af5acbf4..30d7409d0 100644 --- a/tests/llama_stack/safety/conftest.py +++ b/tests/llama_stack/safety/conftest.py @@ -54,30 +54,56 @@ def guardrails_orchestrator_ssl_cert(guardrails_orchestrator_route: Route): @pytest.fixture(scope="class") def guardrails_orchestrator_ssl_cert_secret( + pytestconfig: pytest.Config, admin_client: DynamicClient, model_namespace: Namespace, guardrails_orchestrator_ssl_cert: str, # ← Add dependency and use correct cert -) -> Generator[Secret, Any, None]: + teardown_resources: bool, +) -> Generator[Secret, Any, Any]: with open(guardrails_orchestrator_ssl_cert, "r") as f: cert_content = f.read() - with Secret( + secret = Secret( client=admin_client, name="orch-certificate", namespace=model_namespace.name, data_dict={"orch-certificate.crt": b64encode(cert_content.encode("utf-8")).decode("utf-8")}, - ) as secret: + ensure_exists=pytestconfig.option.post_upgrade, + teardown=teardown_resources, + ) + if pytestconfig.option.post_upgrade and pytestconfig.option.teardown_resources: yield secret + secret.clean_up() + else: + with secret: + yield secret @pytest.fixture(scope="class") -def patched_llamastack_deployment_tls_certs(llama_stack_distribution, guardrails_orchestrator_ssl_cert_secret): +def patched_llamastack_deployment_tls_certs( + pytestconfig: pytest.Config, + llama_stack_distribution, + guardrails_orchestrator_ssl_cert_secret, +): lls_deployment = Deployment( name=llama_stack_distribution.name, namespace=llama_stack_distribution.namespace, ensure_exists=True, ) + if pytestconfig.option.post_upgrade: + current_spec = lls_deployment.instance.spec.template.spec.to_dict() + volumes = current_spec.get("volumes", []) + container_spec = next((c for c in current_spec.get("containers", []) if c.get("name") == "llama-stack"), {}) + volume_mounts = container_spec.get("volumeMounts", []) + + has_router_ca_volume = any(v.get("name") == "router-ca" for v in volumes) + has_router_ca_mount = any(vm.get("name") == "router-ca" for vm in volume_mounts) + if not (has_router_ca_volume and has_router_ca_mount): + raise ValueError("Expected existing router-ca TLS cert patch to be present in post-upgrade run") + yield lls_deployment + return + current_spec = lls_deployment.instance.spec.template.spec.to_dict() current_spec["volumes"].append({ diff --git a/tests/llama_stack/vector_io/upgrade/test_upgrade_vector_store_rag.py b/tests/llama_stack/vector_io/upgrade/test_upgrade_vector_store_rag.py new file mode 100644 index 000000000..64505ae48 --- /dev/null +++ b/tests/llama_stack/vector_io/upgrade/test_upgrade_vector_store_rag.py @@ -0,0 +1,128 @@ +import pytest +from llama_stack_client import LlamaStackClient +from llama_stack_client.types.vector_store import VectorStore + +from tests.llama_stack.constants import ModelInfo + +IBM_EARNINGS_RAG_QUERY = "How did IBM perform financially in the fourth quarter of 2025?" + + +def _assert_minimal_rag_response( + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, + vector_store_with_example_docs: VectorStore, +) -> None: + response = unprivileged_llama_stack_client.responses.create( + input=IBM_EARNINGS_RAG_QUERY, + model=llama_stack_models.model_id, + instructions="Always use the file_search tool to look up information before answering.", + stream=False, + tools=[ + { + "type": "file_search", + "vector_store_ids": [vector_store_with_example_docs.id], + } + ], + ) + + file_search_calls = [item for item in response.output if item.type == "file_search_call"] + assert file_search_calls, ( + "Expected file_search_call output item in the response, indicating the model " + f"invoked file_search. Output types: {[item.type for item in response.output]}" + ) + + file_search_call = file_search_calls[0] + assert file_search_call.status == "completed", ( + f"Expected file_search_call status 'completed', got '{file_search_call.status}'" + ) + assert file_search_call.results, "file_search_call should contain retrieval results" + + annotations = [] + for item in response.output: + if item.type != "message" or not isinstance(item.content, list): + continue + for content_item in item.content: + item_annotations = getattr(content_item, "annotations", None) + if item_annotations: + annotations.extend(item_annotations) + + assert annotations, "Response should contain file_citation annotations when file_search returns results" + assert any(annotation.type == "file_citation" for annotation in annotations), ( + "Expected at least one file_citation annotation in response output" + ) + + +@pytest.mark.parametrize( + "unprivileged_model_namespace, llama_stack_server_config, vector_store", + [ + pytest.param( + {"name": "test-llamastack-vector-rag-upgrade"}, + { + "llama_stack_storage_size": "2Gi", + "vector_io_provider": "milvus", + "files_provider": "s3", + }, + {"vector_io_provider": "milvus"}, + ), + ], + indirect=True, +) +@pytest.mark.llama_stack +@pytest.mark.rag +class TestPreUpgradeLlamaStackVectorStoreRag: + @pytest.mark.pre_upgrade + def test_vector_store_rag_pre_upgrade( + self, + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, + vector_store_with_example_docs: VectorStore, + ) -> None: + """Verify vector-store-backed RAG works before upgrade. + + Given: A running unprivileged LlamaStack distribution with a vector store and uploaded documents. + When: A retrieval-augmented response is requested using file search. + Then: The response includes completed file_search_call output and file_citation annotations. + """ + _assert_minimal_rag_response( + unprivileged_llama_stack_client=unprivileged_llama_stack_client, + llama_stack_models=llama_stack_models, + vector_store_with_example_docs=vector_store_with_example_docs, + ) + + +@pytest.mark.parametrize( + "unprivileged_model_namespace, llama_stack_server_config, vector_store", + [ + pytest.param( + {"name": "test-llamastack-vector-rag-upgrade"}, + { + "llama_stack_storage_size": "2Gi", + "vector_io_provider": "milvus", + "files_provider": "s3", + }, + {"vector_io_provider": "milvus"}, + ), + ], + indirect=True, +) +@pytest.mark.llama_stack +@pytest.mark.rag +class TestPostUpgradeLlamaStackVectorStoreRag: + @pytest.mark.post_upgrade + def test_vector_store_rag_post_upgrade( + self, + unprivileged_llama_stack_client: LlamaStackClient, + llama_stack_models: ModelInfo, + vector_store_with_example_docs: VectorStore, + ) -> None: + """Verify vector-store-backed RAG remains correct after upgrade. + + Given: A pre-existing unprivileged LlamaStack distribution after upgrade with reused vector store docs. + When: A retrieval-augmented response is requested using file search. + Then: The response includes completed file_search_call output and file_citation annotations. + """ + _assert_minimal_rag_response( + unprivileged_llama_stack_client=unprivileged_llama_stack_client, + llama_stack_models=llama_stack_models, + vector_store_with_example_docs=vector_store_with_example_docs, + )