Merge branch 'main' into feature/api-key-authorization-tests

Author: dbasunag

docs/GETTING_STARTED.md

@@ -138,7 +138,7 @@ To run tests with admin client only, pass `--tc=use_unprivileged_client:False` t
 ### jira integration
 
 To skip running tests which have open bugs, [pytest_jira](https://github.com/rhevm-qe-automation/pytest_jira) plugin is used.
-To run tests with jira integration, you need to set `PYTEST_JIRA_URL` and `PYTEST_JIRA_TOKEN` environment variables.
+To run tests with jira integration, you need to set `PYTEST_JIRA_URL`, `PYTEST_JIRA_USERNAME` and `PYTEST_JIRA_TOKEN` environment variables.
 To make a test with jira marker, add: `@pytest.mark.jira(jira_id="RHOAIENG-0000", run=False)` to the test.
 
 ### Running containerized tests

pyproject.toml

@@ -70,7 +70,7 @@ dependencies = [
     "semver>=3.0.4",
     "sqlalchemy>=2.0.40",
     "pytest-order>=1.3.0",
-    "marshmallow==3.26.2,<4", # this version is needed for pytest-jira
+    "marshmallow>=4.0",
     "pytest-html>=4.1.1",
     "fire",
     "llama_stack_client>=0.6.0,<0.7",

pytest.ini

@@ -64,3 +64,5 @@ addopts =
     --tc-file=tests/global_config.py
     --tc-format=python
     --jira
+    --jira-connection-error-strategy=skip
+    --jira-disable-docs-search

tests/conftest.py

@@ -554,13 +554,29 @@ def cluster_monitoring_config(
 
 @pytest.fixture(scope="class")
 def unprivileged_model_namespace(
-    request: FixtureRequest, admin_client: DynamicClient, unprivileged_client: DynamicClient
+    request: FixtureRequest,
+    pytestconfig: pytest.Config,
+    admin_client: DynamicClient,
+    unprivileged_client: DynamicClient,
+    teardown_resources: bool,
 ) -> Generator[Namespace, Any, Any]:
     if request.param.get("modelmesh-enabled"):
         request.getfixturevalue(argname="enabled_modelmesh_in_dsc")
 
-    with create_ns(admin_client=admin_client, unprivileged_client=unprivileged_client, pytest_request=request) as ns:
+    ns = Namespace(client=unprivileged_client, name=request.param["name"])
+    if pytestconfig.option.post_upgrade:
         yield ns
+        ns.client = admin_client
+        if teardown_resources:
+            ns.clean_up()
+    else:
+        with create_ns(
+            admin_client=admin_client,
+            unprivileged_client=unprivileged_client,
+            pytest_request=request,
+            teardown=teardown_resources,
+        ) as ns:
+            yield ns
 
 
 # MinIo

tests/fixtures/files.py

@@ -64,11 +64,13 @@ def _factory(provider_name: str) -> list[dict[str, str]]:
             })
             env_vars.append({
                 "name": "AWS_ACCESS_KEY_ID",
-                "value": request.getfixturevalue(argname="aws_access_key_id"),
+                "valueFrom": {"secretKeyRef": {"name": "llamastack-distribution-secret", "key": "aws-access-key-id"}},
             })
             env_vars.append({
                 "name": "AWS_SECRET_ACCESS_KEY",
-                "value": request.getfixturevalue(argname="aws_secret_access_key"),
+                "valueFrom": {
+                    "secretKeyRef": {"name": "llamastack-distribution-secret", "key": "aws-secret-access-key"}
+                },
             })
             env_vars.append({"name": "S3_AUTO_CREATE_BUCKET", "value": S3_AUTO_CREATE_BUCKET})
 

tests/fixtures/vector_io.py

@@ -151,11 +151,13 @@ def _factory(provider_name: str) -> list[dict[str, Any]]:
 
 @pytest.fixture(scope="class")
 def vector_io_secret(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
+    teardown_resources: bool,
 ) -> Generator[Secret, Any, Any]:
     """Create a secret for the vector I/O providers"""
-    with Secret(
+    secret = Secret(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-secret",
@@ -166,37 +168,55 @@ def vector_io_secret(
             "pgvector-password": PGVECTOR_PASSWORD,
             "milvus-token": MILVUS_TOKEN,
         },
-    ) as secret:
+        ensure_exists=pytestconfig.option.post_upgrade,
+        teardown=teardown_resources,
+    )
+    if pytestconfig.option.post_upgrade:
         yield secret
+        secret.clean_up()
+    else:
+        with secret:
+            yield secret
 
 
 @pytest.fixture(scope="class")
 def etcd_deployment(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
+    teardown_resources: bool,
 ) -> Generator[Deployment, Any, Any]:
     """Deploy an etcd instance for vector I/O provider testing."""
-    with Deployment(
+    deployment = Deployment(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-etcd-deployment",
         replicas=1,
         selector={"matchLabels": {"app": "etcd"}},
         strategy={"type": "Recreate"},
         template=get_etcd_deployment_template(),
-        teardown=True,
-    ) as deployment:
+        teardown=teardown_resources,
+        ensure_exists=pytestconfig.option.post_upgrade,
+    )
+    if pytestconfig.option.post_upgrade:
         deployment.wait_for_replicas(deployed=True, timeout=120)
         yield deployment
+        deployment.clean_up()
+    else:
+        with deployment:
+            deployment.wait_for_replicas(deployed=True, timeout=120)
+            yield deployment
 
 
 @pytest.fixture(scope="class")
 def etcd_service(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
+    teardown_resources: bool,
 ) -> Generator[Service, Any, Any]:
     """Create a service for the etcd deployment."""
-    with Service(
+    service = Service(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-etcd-service",
@@ -208,24 +228,33 @@ def etcd_service(
         ],
         selector={"app": "etcd"},
         wait_for_resource=True,
-    ) as service:
+        ensure_exists=pytestconfig.option.post_upgrade,
+        teardown=teardown_resources,
+    )
+    if pytestconfig.option.post_upgrade:
         yield service
+        service.clean_up()
+    else:
+        with service:
+            yield service
 
 
 @pytest.fixture(scope="class")
 def remote_milvus_deployment(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     etcd_deployment: Deployment,
     etcd_service: Service,
     vector_io_secret: Secret,
+    teardown_resources: bool,
 ) -> Generator[Deployment, Any, Any]:
     """Deploy a remote Milvus instance for vector I/O provider testing."""
     _ = etcd_deployment
     _ = etcd_service
     _ = vector_io_secret
 
-    with Deployment(
+    deployment = Deployment(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-milvus-deployment",
@@ -234,22 +263,31 @@ def remote_milvus_deployment(
         selector={"matchLabels": {"app": "milvus-standalone"}},
         strategy={"type": "Recreate"},
         template=get_milvus_deployment_template(),
-        teardown=True,
-    ) as deployment:
+        teardown=teardown_resources,
+        ensure_exists=pytestconfig.option.post_upgrade,
+    )
+    if pytestconfig.option.post_upgrade:
         deployment.wait_for_replicas(deployed=True, timeout=240)
         yield deployment
+        deployment.clean_up()
+    else:
+        with deployment:
+            deployment.wait_for_replicas(deployed=True, timeout=240)
+            yield deployment
 
 
 @pytest.fixture(scope="class")
 def milvus_service(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     remote_milvus_deployment: Deployment,
+    teardown_resources: bool,
 ) -> Generator[Service, Any, Any]:
     """Create a service for the remote Milvus deployment."""
     _ = remote_milvus_deployment
 
-    with Service(
+    service = Service(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-milvus-service",
@@ -262,8 +300,15 @@ def milvus_service(
         ],
         selector={"app": "milvus-standalone"},
         wait_for_resource=True,
-    ) as service:
+        ensure_exists=pytestconfig.option.post_upgrade,
+        teardown=teardown_resources,
+    )
+    if pytestconfig.option.post_upgrade:
         yield service
+        service.clean_up()
+    else:
+        with service:
+            yield service
 
 
 def get_milvus_deployment_template() -> dict[str, Any]:
@@ -343,14 +388,16 @@ def get_etcd_deployment_template() -> dict[str, Any]:
 
 @pytest.fixture(scope="class")
 def pgvector_deployment(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     vector_io_secret: Secret,
+    teardown_resources: bool,
 ) -> Generator[Deployment, Any, Any]:
     """Deploy a PGVector instance for vector I/O provider testing."""
     _ = vector_io_secret
 
-    with Deployment(
+    deployment = Deployment(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-pgvector-deployment",
@@ -359,22 +406,31 @@ def pgvector_deployment(
         selector={"matchLabels": {"app": "pgvector"}},
         strategy={"type": "Recreate"},
         template=get_pgvector_deployment_template(),
-        teardown=True,
-    ) as deployment:
+        teardown=teardown_resources,
+        ensure_exists=pytestconfig.option.post_upgrade,
+    )
+    if pytestconfig.option.post_upgrade:
         deployment.wait_for_replicas(deployed=True, timeout=240)
         yield deployment
+        deployment.clean_up()
+    else:
+        with deployment:
+            deployment.wait_for_replicas(deployed=True, timeout=240)
+            yield deployment
 
 
 @pytest.fixture(scope="class")
 def pgvector_service(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     pgvector_deployment: Deployment,
+    teardown_resources: bool,
 ) -> Generator[Service, Any, Any]:
     """Create a service for the PGVector deployment."""
     _ = pgvector_deployment
 
-    with Service(
+    service = Service(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-pgvector-service",
@@ -387,8 +443,15 @@ def pgvector_service(
         ],
         selector={"app": "pgvector"},
         wait_for_resource=True,
-    ) as service:
+        ensure_exists=pytestconfig.option.post_upgrade,
+        teardown=teardown_resources,
+    )
+    if pytestconfig.option.post_upgrade:
         yield service
+        service.clean_up()
+    else:
+        with service:
+            yield service
 
 
 def get_pgvector_deployment_template() -> dict[str, Any]:
@@ -438,14 +501,16 @@ def get_pgvector_deployment_template() -> dict[str, Any]:
 
 @pytest.fixture(scope="class")
 def qdrant_deployment(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     vector_io_secret: Secret,
+    teardown_resources: bool,
 ) -> Generator[Deployment, Any, Any]:
     """Deploy a Qdrant instance for vector I/O provider testing."""
     _ = vector_io_secret
 
-    with Deployment(
+    deployment = Deployment(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-qdrant-deployment",
@@ -454,22 +519,31 @@ def qdrant_deployment(
         selector={"matchLabels": {"app": "qdrant"}},
         strategy={"type": "Recreate"},
         template=get_qdrant_deployment_template(),
-        teardown=True,
-    ) as deployment:
+        teardown=teardown_resources,
+        ensure_exists=pytestconfig.option.post_upgrade,
+    )
+    if pytestconfig.option.post_upgrade:
         deployment.wait_for_replicas(deployed=True, timeout=240)
         yield deployment
+        deployment.clean_up()
+    else:
+        with deployment:
+            deployment.wait_for_replicas(deployed=True, timeout=240)
+            yield deployment
 
 
 @pytest.fixture(scope="class")
 def qdrant_service(
+    pytestconfig: pytest.Config,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     qdrant_deployment: Deployment,
+    teardown_resources: bool,
 ) -> Generator[Service, Any, Any]:
     """Create a service for the Qdrant deployment."""
     _ = qdrant_deployment
 
-    with Service(
+    service = Service(
         client=unprivileged_client,
         namespace=unprivileged_model_namespace.name,
         name="vector-io-qdrant-service",
@@ -487,8 +561,15 @@ def qdrant_service(
         ],
         selector={"app": "qdrant"},
         wait_for_resource=True,
-    ) as service:
+        ensure_exists=pytestconfig.option.post_upgrade,
+        teardown=teardown_resources,
+    )
+    if pytestconfig.option.post_upgrade:
         yield service
+        service.clean_up()
+    else:
+        with service:
+            yield service
 
 
 def get_qdrant_deployment_template() -> dict[str, Any]: