From 2c529b0553c58fa5ee4de7362b6d9575aecfa1fb Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 13:53:59 +0300 Subject: [PATCH 1/7] feat: use unprivileged_client --- docs/GETTING_STARTED.md | 4 ++ tests/conftest.py | 8 ++- tests/global_config.py | 2 + .../model_registry/negative_tests/conftest.py | 2 +- .../model_server/authentication/conftest.py | 69 +++++++++---------- .../model_server/components/conftest.py | 4 +- .../kserve_dsc_deployment_mode/conftest.py | 4 +- tests/model_serving/model_server/conftest.py | 60 ++++++++-------- .../conftest.py | 12 ++-- .../test_isvc_replicas_update.py | 4 +- .../model_server/model_car/conftest.py | 4 +- .../model_server/multi_node/conftest.py | 28 ++++---- .../multi_node/test_nvidia_multi_node.py | 12 ++-- .../model_server/ovms/model_mesh/conftest.py | 9 ++- .../model_server/private_endpoint/conftest.py | 24 +++---- .../model_server/private_endpoint/utils.py | 6 +- .../test_kserve_raw_routes_reconciliation.py | 6 +- .../model_server/raw_deployment/utils.py | 6 +- .../model_server/routes/conftest.py | 2 +- .../runtime_configuration/conftest.py | 4 +- .../model_server/serverless/conftest.py | 8 +-- .../serverless/test_concurrency_auto_scale.py | 8 +-- .../serverless/test_scale_to_zero.py | 12 ++-- .../model_server/storage/minio/conftest.py | 12 ++-- .../model_server/storage/pvc/conftest.py | 18 ++--- .../pvc/test_kserve_pvc_write_access.py | 8 +-- .../model_server/upgrade/conftest.py | 62 ++++++++--------- utilities/general.py | 8 +-- utilities/infra.py | 10 +-- 29 files changed, 212 insertions(+), 204 deletions(-) diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index a99a252f2..5bc855fac 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -67,6 +67,10 @@ By default, cluster sanity checks are run to make cluster ready for tests. To skip cluster sanity checks, pass `--cluster-sanity-skip-check` to skip all tests. To skip RHOAI/ODH-related tests (for example when running in upstream), pass `--cluster-sanity-skip-rhoai-check`. +### Running tests with admin client instead of unprivileged client +To run tests with admin client only, pass `--tc=use_unprivileged_client:False` to pytest. + + ### jira integration To skip running tests which have open bugs, [pytest_jira](https://github.com/rhevm-qe-automation/pytest_jira) plugin is used. To run tests with jira integration, you need to set `PYTEST_JIRA_URL` and `PYTEST_JIRA_TOKEN` environment variables. diff --git a/tests/conftest.py b/tests/conftest.py index dc25c530b..e7e1d6ffb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,7 +111,7 @@ def model_namespace( ns.clean_up() else: with create_ns( - admin_client=admin_client, + client=admin_client, pytest_request=request, teardown=teardown_resources, ) as ns: @@ -275,6 +275,10 @@ def unprivileged_client( """ Provides none privileged API client. If non_admin_user_password is None, then it will raise. """ + if not py_config.get("use_unprivileged_client"): + LOGGER.warning("Unprivileged client is not enabled, using admin client") + yield admin_client + if non_admin_user_password is None: raise ValueError("Unprivileged user not provisioned") @@ -380,7 +384,7 @@ def unprivileged_model_namespace( def minio_namespace(admin_client: DynamicClient) -> Generator[Namespace, Any, Any]: with create_ns( name=f"{MinIo.Metadata.NAME}-{shortuuid.uuid().lower()}", - admin_client=admin_client, + client=admin_client, ) as ns: yield ns diff --git a/tests/global_config.py b/tests/global_config.py index b5b6e9ef7..bf1dfc7dd 100644 --- a/tests/global_config.py +++ b/tests/global_config.py @@ -5,6 +5,8 @@ dsc_name: str = "default-dsc" dsci_name: str = "default-dsci" +use_unprivileged_client: bool = True + for _dir in dir(): val = locals()[_dir] if type(val) not in [bool, list, dict, str, int]: diff --git a/tests/model_registry/negative_tests/conftest.py b/tests/model_registry/negative_tests/conftest.py index a57ea9a35..7aef950fc 100644 --- a/tests/model_registry/negative_tests/conftest.py +++ b/tests/model_registry/negative_tests/conftest.py @@ -27,7 +27,7 @@ def model_registry_namespace_for_negative_tests( ) -> Generator[Namespace, Any, Any]: with create_ns( name=request.param.get("namespace_name", CUSTOM_NEGATIVE_NS), - admin_client=admin_client, + client=admin_client, ) as ns: yield ns diff --git a/tests/model_serving/model_server/authentication/conftest.py b/tests/model_serving/model_server/authentication/conftest.py index ddb490577..8804b98ee 100644 --- a/tests/model_serving/model_server/authentication/conftest.py +++ b/tests/model_serving/model_server/authentication/conftest.py @@ -34,10 +34,10 @@ # GRPC model serving @pytest.fixture(scope="class") def grpc_model_service_account( - admin_client: DynamicClient, models_endpoint_s3_secret: Secret + unprivileged_client: DynamicClient, models_endpoint_s3_secret: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( - client=admin_client, + client=unprivileged_client, namespace=models_endpoint_s3_secret.namespace, name=f"{Protocols.GRPC}-models-bucket-sa", secrets=[{"name": models_endpoint_s3_secret.name}], @@ -47,11 +47,11 @@ def grpc_model_service_account( @pytest.fixture(scope="class") def grpc_s3_caikit_serving_runtime( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.GRPC}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}", namespace=model_namespace.name, template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING, @@ -64,14 +64,14 @@ def grpc_s3_caikit_serving_runtime( @pytest.fixture(scope="class") def grpc_s3_inference_service( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, grpc_s3_caikit_serving_runtime: ServingRuntime, s3_models_storage_uri: str, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.GRPC}-{ModelFormat.CAIKIT}", namespace=model_namespace.name, runtime=grpc_s3_caikit_serving_runtime.name, @@ -86,11 +86,11 @@ def grpc_s3_inference_service( @pytest.fixture(scope="class") def http_view_role( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_s3_caikit_serverless_inference_service: InferenceService, ) -> Generator[Role, Any, Any]: with create_isvc_view_role( - client=admin_client, + client=unprivileged_client, isvc=http_s3_caikit_serverless_inference_service, name=f"{http_s3_caikit_serverless_inference_service.name}-view", resource_names=[http_s3_caikit_serverless_inference_service.name], @@ -100,11 +100,11 @@ def http_view_role( @pytest.fixture(scope="class") def http_raw_view_role( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_s3_caikit_raw_inference_service: InferenceService, ) -> Generator[Role, Any, Any]: with create_isvc_view_role( - client=admin_client, + client=unprivileged_client, isvc=http_s3_caikit_raw_inference_service, name=f"{http_s3_caikit_raw_inference_service.name}-view", resource_names=[http_s3_caikit_raw_inference_service.name], @@ -114,13 +114,13 @@ def http_raw_view_role( @pytest.fixture(scope="class") def http_role_binding( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_view_role: Role, model_service_account: ServiceAccount, http_s3_caikit_serverless_inference_service: InferenceService, ) -> Generator[RoleBinding, Any, Any]: with RoleBinding( - client=admin_client, + client=unprivileged_client, namespace=model_service_account.namespace, name=f"{Protocols.HTTP}-{model_service_account.name}-view", role_ref_name=http_view_role.name, @@ -133,13 +133,13 @@ def http_role_binding( @pytest.fixture(scope="class") def http_raw_role_binding( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_raw_view_role: Role, model_service_account: ServiceAccount, http_s3_caikit_raw_inference_service: InferenceService, ) -> Generator[RoleBinding, Any, Any]: with RoleBinding( - client=admin_client, + client=unprivileged_client, namespace=model_service_account.namespace, name=f"{Protocols.HTTP}-{model_service_account.name}-view", role_ref_name=http_raw_view_role.name, @@ -162,7 +162,6 @@ def http_raw_inference_token(model_service_account: ServiceAccount, http_raw_rol @pytest.fixture() def patched_remove_authentication_isvc( - admin_client: DynamicClient, http_s3_caikit_serverless_inference_service: InferenceService, ) -> Generator[InferenceService, Any, Any]: with ResourceEditor( @@ -180,10 +179,11 @@ def patched_remove_authentication_isvc( @pytest.fixture() def patched_remove_raw_authentication_isvc( admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_s3_caikit_raw_inference_service: InferenceService, ) -> Generator[InferenceService, Any, Any]: predictor_pod = get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=http_s3_caikit_raw_inference_service, )[0] @@ -204,10 +204,10 @@ def patched_remove_raw_authentication_isvc( @pytest.fixture(scope="class") def model_service_account_2( - admin_client: DynamicClient, models_endpoint_s3_secret: Secret + unprivileged_client: DynamicClient, models_endpoint_s3_secret: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( - client=admin_client, + client=unprivileged_client, namespace=models_endpoint_s3_secret.namespace, name="models-bucket-sa-2", secrets=[{"name": models_endpoint_s3_secret.name}], @@ -217,10 +217,10 @@ def model_service_account_2( @pytest.fixture(scope="class") def grpc_view_role( - admin_client: DynamicClient, grpc_s3_inference_service: InferenceService + unprivileged_client: DynamicClient, grpc_s3_inference_service: InferenceService ) -> Generator[Role, Any, Any]: with create_isvc_view_role( - client=admin_client, + client=unprivileged_client, isvc=grpc_s3_inference_service, name=f"{grpc_s3_inference_service.name}-view", resource_names=[grpc_s3_inference_service.name], @@ -230,13 +230,13 @@ def grpc_view_role( @pytest.fixture(scope="class") def grpc_role_binding( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, grpc_view_role: Role, grpc_model_service_account: ServiceAccount, grpc_s3_inference_service: InferenceService, ) -> Generator[RoleBinding, Any, Any]: with RoleBinding( - client=admin_client, + client=unprivileged_client, namespace=grpc_model_service_account.namespace, name=f"{Protocols.GRPC}-{grpc_model_service_account.name}-view", role_ref_name=grpc_view_role.name, @@ -255,14 +255,14 @@ def grpc_inference_token(grpc_model_service_account: ServiceAccount, grpc_role_b @pytest.fixture(scope="class") def http_s3_caikit_serverless_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}", namespace=model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, @@ -278,7 +278,7 @@ def http_s3_caikit_serverless_inference_service( @pytest.fixture(scope="class") def http_s3_caikit_raw_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, @@ -286,7 +286,7 @@ def http_s3_caikit_raw_inference_service( model_service_account: ServiceAccount, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}", namespace=model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, @@ -304,14 +304,14 @@ def http_s3_caikit_raw_inference_service( @pytest.fixture(scope="class") def http_s3_caikit_raw_inference_service_2( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, model_service_account_2: ServiceAccount, ) -> InferenceService: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}-2", namespace=model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, @@ -328,11 +328,11 @@ def http_s3_caikit_raw_inference_service_2( @pytest.fixture(scope="class") def http_s3_caikit_tgis_serving_runtime( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}", namespace=model_namespace.name, template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING, @@ -345,7 +345,6 @@ def http_s3_caikit_tgis_serving_runtime( @pytest.fixture() def patched_remove_authentication_model_mesh_runtime( - admin_client: DynamicClient, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ) -> Generator[ServingRuntime, Any, Any]: with ResourceEditor( @@ -362,12 +361,12 @@ def patched_remove_authentication_model_mesh_runtime( @pytest.fixture(scope="class") def http_model_mesh_view_role( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_s3_openvino_model_mesh_inference_service: InferenceService, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ) -> Generator[Role, Any, Any]: with Role( - client=admin_client, + client=unprivileged_client, name=f"{http_s3_openvino_model_mesh_inference_service.name}-view", namespace=http_s3_openvino_model_mesh_inference_service.namespace, rules=[ @@ -379,12 +378,12 @@ def http_model_mesh_view_role( @pytest.fixture(scope="class") def http_model_mesh_role_binding( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_model_mesh_view_role: Role, ci_service_account: ServiceAccount, ) -> Generator[RoleBinding, Any, Any]: with RoleBinding( - client=admin_client, + client=unprivileged_client, namespace=ci_service_account.namespace, name=f"{Protocols.HTTP}-{ci_service_account.name}-view", role_ref_name=http_model_mesh_view_role.name, diff --git a/tests/model_serving/model_server/components/conftest.py b/tests/model_serving/model_server/components/conftest.py index d23b66d03..bec687a1f 100644 --- a/tests/model_serving/model_server/components/conftest.py +++ b/tests/model_serving/model_server/components/conftest.py @@ -33,14 +33,14 @@ def managed_modelmesh_kserve_in_dsc( @pytest.fixture(scope="class") def invalid_s3_models_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_s3_bucket_name: str, model_service_account: ServiceAccount, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=model_namespace.name, runtime=serving_runtime_from_template.name, diff --git a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py index 1398c7b24..fbae56297 100644 --- a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py +++ b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py @@ -57,13 +57,13 @@ def patched_default_deployment_mode_in_dsc( @pytest.fixture(scope="class") def ovms_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=model_namespace.name, runtime=ovms_kserve_serving_runtime.name, diff --git a/tests/model_serving/model_server/conftest.py b/tests/model_serving/model_server/conftest.py index 9e2da511a..06970aa69 100644 --- a/tests/model_serving/model_server/conftest.py +++ b/tests/model_serving/model_server/conftest.py @@ -61,7 +61,7 @@ def skip_if_no_deployed_openshift_serverless(admin_client: DynamicClient) -> Non @pytest.fixture(scope="class") def models_endpoint_s3_secret( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -70,7 +70,7 @@ def models_endpoint_s3_secret( models_s3_bucket_endpoint: str, ) -> Generator[Secret, Any, Any]: with s3_endpoint_secret( - client=admin_client, + client=unprivileged_client, name="models-bucket-secret", namespace=model_namespace.name, aws_access_key=aws_access_key_id, @@ -85,10 +85,10 @@ def models_endpoint_s3_secret( # HTTP model serving @pytest.fixture(scope="class") def model_service_account( - admin_client: DynamicClient, models_endpoint_s3_secret: Secret + unprivileged_client: DynamicClient, models_endpoint_s3_secret: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( - client=admin_client, + client=unprivileged_client, namespace=models_endpoint_s3_secret.namespace, name="models-bucket-sa", secrets=[{"name": models_endpoint_s3_secret.name}], @@ -99,11 +99,11 @@ def model_service_account( @pytest.fixture(scope="class") def serving_runtime_from_template( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": request.param["name"], "namespace": model_namespace.name, "template_name": request.param["template-name"], @@ -125,13 +125,13 @@ def serving_runtime_from_template( @pytest.fixture(scope="class") def s3_models_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": request.param["name"], "namespace": model_namespace.name, "runtime": serving_runtime_from_template.name, @@ -160,14 +160,14 @@ def s3_models_inference_service( @pytest.fixture(scope="class") def model_pvc( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[PersistentVolumeClaim, Any, Any]: access_mode = "ReadWriteOnce" pvc_kwargs = { "name": "model-pvc", "namespace": model_namespace.name, - "client": admin_client, + "client": unprivileged_client, "size": request.param["pvc-size"], } if hasattr(request, "param"): @@ -214,14 +214,14 @@ def skip_if_no_deployed_openshift_service_mesh(admin_client: DynamicClient) -> N @pytest.fixture(scope="class") def http_s3_openvino_model_mesh_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.OPENVINO}", namespace=model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, @@ -238,11 +238,11 @@ def http_s3_openvino_model_mesh_inference_service( @pytest.fixture(scope="class") def http_s3_ovms_model_mesh_serving_runtime( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "namespace": model_namespace.name, "name": f"{Protocols.HTTP}-{ModelInferenceRuntime.OPENVINO_RUNTIME}", "template_name": RuntimeTemplates.OVMS_MODEL_MESH, @@ -278,11 +278,11 @@ def http_s3_ovms_model_mesh_serving_runtime( @pytest.fixture(scope="class") def ovms_kserve_serving_runtime( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "namespace": model_namespace.name, "name": request.param["runtime-name"], "template_name": RuntimeTemplates.OVMS_KSERVE, @@ -310,7 +310,7 @@ def ovms_kserve_serving_runtime( @pytest.fixture(scope="class") def ci_endpoint_s3_secret( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -319,7 +319,7 @@ def ci_endpoint_s3_secret( ci_s3_bucket_endpoint: str, ) -> Generator[Secret, Any, Any]: with s3_endpoint_secret( - client=admin_client, + client=unprivileged_client, name="ci-bucket-secret", namespace=model_namespace.name, aws_access_key=aws_access_key_id, @@ -333,10 +333,10 @@ def ci_endpoint_s3_secret( @pytest.fixture(scope="class") def ci_service_account( - admin_client: DynamicClient, ci_endpoint_s3_secret: Secret + unprivileged_client: DynamicClient, ci_endpoint_s3_secret: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( - client=admin_client, + client=unprivileged_client, namespace=ci_endpoint_s3_secret.namespace, name="ci-models-bucket-sa", secrets=[{"name": ci_endpoint_s3_secret.name}], @@ -347,14 +347,14 @@ def ci_service_account( @pytest.fixture(scope="class") def ovms_kserve_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: deployment_mode = request.param["deployment-mode"] isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": f"{request.param['name']}-{deployment_mode.lower()}", "namespace": model_namespace.name, "runtime": ovms_kserve_serving_runtime.name, @@ -389,13 +389,13 @@ def ovms_kserve_inference_service( @pytest.fixture(scope="class") def ovms_raw_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{request.param['name']}-raw", namespace=model_namespace.name, external_route=True, @@ -412,14 +412,14 @@ def ovms_raw_inference_service( @pytest.fixture(scope="class") def http_s3_tensorflow_model_mesh_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.TENSORFLOW}", namespace=model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, @@ -475,11 +475,11 @@ def user_workload_monitoring_config_map( @pytest.fixture(scope="class") def http_s3_ovms_external_route_model_mesh_serving_runtime( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "namespace": model_namespace.name, "name": f"{Protocols.HTTP}-{ModelInferenceRuntime.OPENVINO_RUNTIME}-exposed", "template_name": RuntimeTemplates.OVMS_MODEL_MESH, @@ -504,7 +504,7 @@ def http_s3_ovms_external_route_model_mesh_serving_runtime( @pytest.fixture(scope="class") def http_s3_openvino_second_model_mesh_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, @@ -512,7 +512,7 @@ def http_s3_openvino_second_model_mesh_inference_service( # Dynamically select the used ServingRuntime by passing "runtime-fixture-name" request.param runtime = request.getfixturevalue(argname=request.param["runtime-fixture-name"]) with create_isvc( - client=admin_client, + client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.OPENVINO}-2", namespace=model_namespace.name, runtime=runtime.name, diff --git a/tests/model_serving/model_server/inference_service_configuration/conftest.py b/tests/model_serving/model_server/inference_service_configuration/conftest.py index 0081c7960..05fb5c604 100644 --- a/tests/model_serving/model_server/inference_service_configuration/conftest.py +++ b/tests/model_serving/model_server/inference_service_configuration/conftest.py @@ -17,7 +17,7 @@ @pytest.fixture(scope="class") def removed_isvc_env_vars( request: pytest.FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, ovms_kserve_inference_service: InferenceService, ) -> Generator[InferenceService, Any, Any]: if isvc_predictor_spec_model_env := ovms_kserve_inference_service.instance.spec.predictor.model.get("env"): @@ -26,7 +26,7 @@ def removed_isvc_env_vars( ] with update_inference_service( - client=admin_client, + client=unprivileged_client, isvc=ovms_kserve_inference_service, isvc_updated_dict={"spec": {"predictor": {"model": {"env": isvc_predictor_spec_model_env}}}}, ): @@ -40,19 +40,19 @@ def removed_isvc_env_vars( @pytest.fixture def isvc_pods( - admin_client: DynamicClient, ovms_kserve_inference_service: InferenceService + unprivileged_client: DynamicClient, ovms_kserve_inference_service: InferenceService ) -> Generator[list[Pod], Any, Any]: - yield get_pods_by_isvc_label(client=admin_client, isvc=ovms_kserve_inference_service) + yield get_pods_by_isvc_label(client=unprivileged_client, isvc=ovms_kserve_inference_service) @pytest.fixture(scope="class") def patched_isvc_replicas( request: pytest.FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, ovms_kserve_inference_service: InferenceService, ) -> Generator[InferenceService, Any, Any]: with update_inference_service( - client=admin_client, + client=unprivileged_client, isvc=ovms_kserve_inference_service, isvc_updated_dict={ "spec": { diff --git a/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py b/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py index b06cb55b0..cb1255307 100644 --- a/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py +++ b/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py @@ -64,7 +64,7 @@ def test_raw_increase_isvc_replicas_inference(self, ovms_kserve_inference_servic indirect=True, ) @pytest.mark.dependency(name="test_raw_decrease_isvc_replicas") - def test_raw_decrease_isvc_replicas(self, admin_client, isvc_pods, patched_isvc_replicas): + def test_raw_decrease_isvc_replicas(self, unprivileged_client, isvc_pods, patched_isvc_replicas): """Test replicas decrease""" orig_pod_names = [pod.name for pod in isvc_pods] pods = [] @@ -74,7 +74,7 @@ def test_raw_decrease_isvc_replicas(self, admin_client, isvc_pods, patched_isvc_ wait_timeout=Timeout.TIMEOUT_2MIN, sleep=1, func=get_pods_by_isvc_label, - client=admin_client, + client=unprivileged_client, isvc=patched_isvc_replicas, ): if len(pods) == 1 and pods[0].name in orig_pod_names: diff --git a/tests/model_serving/model_server/model_car/conftest.py b/tests/model_serving/model_server/model_car/conftest.py index d3195a11b..6d23bc905 100644 --- a/tests/model_serving/model_server/model_car/conftest.py +++ b/tests/model_serving/model_server/model_car/conftest.py @@ -14,12 +14,12 @@ @pytest.fixture(scope="class") def model_car_serverless_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name="serverless-model-car", namespace=model_namespace.name, runtime=serving_runtime_from_template.name, diff --git a/tests/model_serving/model_server/multi_node/conftest.py b/tests/model_serving/model_server/multi_node/conftest.py index d0a4c2e96..7cee52489 100644 --- a/tests/model_serving/model_server/multi_node/conftest.py +++ b/tests/model_serving/model_server/multi_node/conftest.py @@ -42,7 +42,7 @@ def skip_if_no_gpu_nodes(nvidia_gpu_nodes: list[Node]) -> None: @pytest.fixture(scope="class") def models_bucket_downloaded_model_data( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, models_s3_bucket_name: str, model_pvc: PersistentVolumeClaim, @@ -52,7 +52,7 @@ def models_bucket_downloaded_model_data( models_s3_bucket_region: str, ) -> str: return download_model_data( - admin_client=admin_client, + client=unprivileged_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, model_namespace=model_namespace.name, @@ -67,11 +67,11 @@ def models_bucket_downloaded_model_data( @pytest.fixture(scope="class") def multi_node_serving_runtime( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( - client=admin_client, + client=unprivileged_client, name="vllm-multinode-runtime", # TODO: rename servingruntime when RHOAIENG-16147 is resolved namespace=model_namespace.name, template_name="vllm-multinode-runtime-template", @@ -84,13 +84,13 @@ def multi_node_serving_runtime( @pytest.fixture(scope="class") def multi_node_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, multi_node_serving_runtime: ServingRuntime, model_pvc: PersistentVolumeClaim, models_bucket_downloaded_model_data: str, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=multi_node_serving_runtime.namespace, runtime=multi_node_serving_runtime.name, @@ -102,7 +102,7 @@ def multi_node_inference_service( wait_for_predictor_pods=False, ) as isvc: wait_for_inference_deployment_replicas( - client=admin_client, + client=unprivileged_client, isvc=isvc, expected_num_deployments=2, runtime_name=multi_node_serving_runtime.name, @@ -112,11 +112,11 @@ def multi_node_inference_service( @pytest.fixture(scope="class") def multi_node_predictor_pods_scope_class( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, multi_node_inference_service: InferenceService, ) -> list[Pod]: return get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=multi_node_inference_service, ) @@ -170,9 +170,9 @@ def ray_ca_tls_secret(admin_client: DynamicClient) -> Secret: @pytest.fixture() -def ray_tls_secret(admin_client: DynamicClient, multi_node_inference_service: InferenceService) -> Secret: +def ray_tls_secret(unprivileged_client: DynamicClient, multi_node_inference_service: InferenceService) -> Secret: return Secret( - client=admin_client, + client=unprivileged_client, name="ray-tls", namespace=multi_node_inference_service.namespace, ) @@ -192,17 +192,17 @@ def deleted_serving_runtime( @pytest.fixture() def deleted_multi_node_pod( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, multi_node_inference_service: InferenceService, ) -> None: delete_multi_node_pod_by_role( - client=admin_client, + client=unprivileged_client, isvc=multi_node_inference_service, role=request.param["pod-role"], ) verify_no_failed_pods( - client=admin_client, + client=unprivileged_client, isvc=multi_node_inference_service, timeout=Timeout.TIMEOUT_10MIN, ) diff --git a/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py b/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py index 64f50684a..43c6b9791 100644 --- a/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py +++ b/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py @@ -105,7 +105,7 @@ def test_cert_files_exist_in_pods(self, multi_node_predictor_pods_scope_class): [pytest.param({"pod-role": HEAD_POD_ROLE})], indirect=True, ) - def test_multi_node_head_pod_deletion(self, admin_client, multi_node_inference_service, deleted_multi_node_pod): + def test_multi_node_head_pod_deletion(self, multi_node_inference_service, deleted_multi_node_pod): """Test multi node when head pod is deleted""" verify_inference_response( inference_service=multi_node_inference_service, @@ -120,7 +120,7 @@ def test_multi_node_head_pod_deletion(self, admin_client, multi_node_inference_s [pytest.param({"pod-role": WORKER_POD_ROLE})], indirect=True, ) - def test_multi_node_worker_pod_deletion(self, admin_client, multi_node_inference_service, deleted_multi_node_pod): + def test_multi_node_worker_pod_deletion(self, multi_node_inference_service, deleted_multi_node_pod): """Test multi node when worker pod is deleted""" verify_inference_response( inference_service=multi_node_inference_service, @@ -172,13 +172,13 @@ def test_multi_node_basic_external_inference(self, patched_multi_node_isvc_exter [pytest.param({"worker-spec": {"pipelineParallelSize": 2, "tensorParallelSize": 4}})], indirect=True, ) - def test_multi_node_tensor_parallel_size_propagation(self, admin_client, patched_multi_node_worker_spec): + def test_multi_node_tensor_parallel_size_propagation(self, unprivileged_client, patched_multi_node_worker_spec): """Test multi node tensor parallel size (number of GPUs per pod) propagation to pod config""" isvc_parallel_size = str(patched_multi_node_worker_spec.instance.spec.predictor.workerSpec.tensorParallelSize) failed_pods: list[dict[str, Any]] = [] - for pod in get_pods_by_isvc_generation(client=admin_client, isvc=patched_multi_node_worker_spec): + for pod in get_pods_by_isvc_generation(client=unprivileged_client, isvc=patched_multi_node_worker_spec): pod_resources = pod.instance.spec.containers[0].resources if not ( isvc_parallel_size @@ -195,10 +195,10 @@ def test_multi_node_tensor_parallel_size_propagation(self, admin_client, patched [pytest.param({"worker-spec": {"pipelineParallelSize": 2, "tensorParallelSize": 1}})], indirect=True, ) - def test_multi_node_pipeline_parallel_size_propagation(self, admin_client, patched_multi_node_worker_spec): + def test_multi_node_pipeline_parallel_size_propagation(self, unprivileged_client, patched_multi_node_worker_spec): """Test multi node pipeline parallel size (number of pods) propagation to pod config""" isvc_parallel_size = patched_multi_node_worker_spec.instance.spec.predictor.workerSpec.pipelineParallelSize - isvc_num_pods = get_pods_by_isvc_generation(client=admin_client, isvc=patched_multi_node_worker_spec) + isvc_num_pods = get_pods_by_isvc_generation(client=unprivileged_client, isvc=patched_multi_node_worker_spec) if isvc_parallel_size != len(isvc_num_pods): pytest.fail( diff --git a/tests/model_serving/model_server/ovms/model_mesh/conftest.py b/tests/model_serving/model_server/ovms/model_mesh/conftest.py index 34372e90f..9114619e1 100644 --- a/tests/model_serving/model_server/ovms/model_mesh/conftest.py +++ b/tests/model_serving/model_server/ovms/model_mesh/conftest.py @@ -16,11 +16,11 @@ @pytest.fixture(scope="class") def model_mesh_view_role( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_s3_openvino_model_mesh_inference_service: ServingRuntime, ) -> Generator[Role, Any, Any]: with create_isvc_view_role( - client=admin_client, + client=unprivileged_client, isvc=http_s3_openvino_model_mesh_inference_service, name=f"{http_s3_openvino_model_mesh_inference_service.name}-view", resource_names=[http_s3_openvino_model_mesh_inference_service.name], @@ -30,12 +30,12 @@ def model_mesh_view_role( @pytest.fixture(scope="class") def model_mesh_role_binding( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_mesh_view_role: Role, ci_service_account: ServiceAccount, ) -> Generator[RoleBinding, Any, Any]: with RoleBinding( - client=admin_client, + client=unprivileged_client, namespace=ci_service_account.namespace, name=f"{Protocols.HTTP}-{ci_service_account.name}-view", role_ref_name=model_mesh_view_role.name, @@ -56,7 +56,6 @@ def model_mesh_inference_token( @pytest.fixture() def patched_model_mesh_sr_with_authentication( - admin_client: DynamicClient, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ) -> Generator[None, None, None]: with ResourceEditor( diff --git a/tests/model_serving/model_server/private_endpoint/conftest.py b/tests/model_serving/model_server/private_endpoint/conftest.py index 4c5aa4060..9a3dee98d 100644 --- a/tests/model_serving/model_server/private_endpoint/conftest.py +++ b/tests/model_serving/model_server/private_endpoint/conftest.py @@ -18,19 +18,19 @@ @pytest.fixture(scope="class") -def diff_namespace(admin_client: DynamicClient) -> Generator[Namespace, Any, Any]: - with create_ns(admin_client=admin_client, name="diff-namespace") as ns: +def diff_namespace(unprivileged_client: DynamicClient) -> Generator[Namespace, Any, Any]: + with create_ns(client=unprivileged_client, name="diff-namespace") as ns: yield ns @pytest.fixture(scope="class") def endpoint_isvc( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, serving_runtime_from_template: ServingRuntime, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name="endpoint-isvc", namespace=serving_runtime_from_template.namespace, deployment_mode=KServeDeploymentType.SERVERLESS, @@ -45,10 +45,10 @@ def endpoint_isvc( @pytest.fixture() def endpoint_pod_with_istio_sidecar( - admin_client: DynamicClient, model_namespace: Namespace + unprivileged_client: DynamicClient, model_namespace: Namespace ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( - admin_client=admin_client, + client=unprivileged_client, namespace=model_namespace.name, use_istio=True, pod_name="test-with-istio", @@ -58,10 +58,10 @@ def endpoint_pod_with_istio_sidecar( @pytest.fixture() def endpoint_pod_without_istio_sidecar( - admin_client: DynamicClient, model_namespace: Namespace + unprivileged_client: DynamicClient, model_namespace: Namespace ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( - admin_client=admin_client, + client=unprivileged_client, namespace=model_namespace.name, use_istio=False, pod_name="test", @@ -71,11 +71,11 @@ def endpoint_pod_without_istio_sidecar( @pytest.fixture() def diff_pod_with_istio_sidecar( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, diff_namespace: Namespace, ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( - admin_client=admin_client, + client=unprivileged_client, namespace=diff_namespace.name, use_istio=True, pod_name="test-with-istio", @@ -85,11 +85,11 @@ def diff_pod_with_istio_sidecar( @pytest.fixture() def diff_pod_without_istio_sidecar( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, diff_namespace: Namespace, ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( - admin_client=admin_client, + client=unprivileged_client, namespace=diff_namespace.name, use_istio=False, pod_name="test", diff --git a/tests/model_serving/model_server/private_endpoint/utils.py b/tests/model_serving/model_server/private_endpoint/utils.py index 51201988c..e2f0f8ec3 100644 --- a/tests/model_serving/model_server/private_endpoint/utils.py +++ b/tests/model_serving/model_server/private_endpoint/utils.py @@ -44,7 +44,7 @@ def curl_from_pod( @contextmanager def create_sidecar_pod( - admin_client: DynamicClient, + client: DynamicClient, namespace: str, use_istio: bool, pod_name: str, @@ -53,7 +53,7 @@ def create_sidecar_pod( Create a sidecar pod Args: - admin_client (DynamicClient): DynamicClient object + client (DynamicClient): DynamicClient object namespace (str): namespace name use_istio (bool): use istio pod_name (str): pod name @@ -76,7 +76,7 @@ def create_sidecar_pod( } ] - pod_kwargs = {"client": admin_client, "name": pod_name, "namespace": namespace, "containers": containers} + pod_kwargs = {"client": client, "name": pod_name, "namespace": namespace, "containers": containers} if use_istio: pod_kwargs.update({"annotations": {"sidecar.istio.io/inject": "true"}}) diff --git a/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py b/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py index 572c958b2..f3e80d46d 100644 --- a/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py +++ b/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py @@ -25,7 +25,7 @@ class TestONNXRawRouteReconciliation: """Test suite for Validating reconciliation""" @pytest.mark.smoke - def test_raw_onnx_rout_reconciliation(self, admin_client, ovms_raw_inference_service): + def test_raw_onnx_rout_reconciliation(self, ovms_raw_inference_service): """ Verify that the KServe Raw ONNX model can be queried using REST and ensure that the model rout reconciliation works correctly . @@ -39,9 +39,9 @@ def test_raw_onnx_rout_reconciliation(self, admin_client, ovms_raw_inference_ser use_default_query=True, ) - def test_route_value_before_and_after_deletion(self, admin_client, ovms_raw_inference_service): + def test_route_value_before_and_after_deletion(self, unprivileged_client, ovms_raw_inference_service): # Validate ingress status before and after route deletion - assert_ingress_status_changed(admin_client=admin_client, inference_service=ovms_raw_inference_service) + assert_ingress_status_changed(client=unprivileged_client, inference_service=ovms_raw_inference_service) def test_model_works_after_route_is_recreated(self, ovms_raw_inference_service): # Final inference validation after route update diff --git a/tests/model_serving/model_server/raw_deployment/utils.py b/tests/model_serving/model_server/raw_deployment/utils.py index 82e60b8a0..84a7671af 100644 --- a/tests/model_serving/model_server/raw_deployment/utils.py +++ b/tests/model_serving/model_server/raw_deployment/utils.py @@ -5,12 +5,12 @@ from utilities.infra import get_model_route -def assert_ingress_status_changed(admin_client: DynamicClient, inference_service: InferenceService) -> None: +def assert_ingress_status_changed(client: DynamicClient, inference_service: InferenceService) -> None: """ Validates that the ingress status changes correctly after route deletion. Args: - admin_client (DynamicClient): The administrative client used to manage the model route. + client (DynamicClient): The administrative client used to manage the model route. inference_service (InferenceService): The inference service whose route status is being checked. Raises: @@ -20,7 +20,7 @@ def assert_ingress_status_changed(admin_client: DynamicClient, inference_service Returns: None """ - route = get_model_route(client=admin_client, isvc=inference_service) + route = get_model_route(client=client, isvc=inference_service) if not route.exists: raise ResourceNotFoundError("Route before deletion not found: No active route is currently available.") diff --git a/tests/model_serving/model_server/routes/conftest.py b/tests/model_serving/model_server/routes/conftest.py index ffb9dfbfe..ea8f4de17 100644 --- a/tests/model_serving/model_server/routes/conftest.py +++ b/tests/model_serving/model_server/routes/conftest.py @@ -16,7 +16,7 @@ @pytest.fixture() def patched_s3_caikit_kserve_isvc_visibility_label( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, s3_models_inference_service: InferenceService, ) -> Generator[InferenceService, Any, Any]: visibility = request.param["visibility"] diff --git a/tests/model_serving/model_server/runtime_configuration/conftest.py b/tests/model_serving/model_server/runtime_configuration/conftest.py index 96400cf42..7229954d3 100644 --- a/tests/model_serving/model_server/runtime_configuration/conftest.py +++ b/tests/model_serving/model_server/runtime_configuration/conftest.py @@ -14,13 +14,13 @@ @pytest.fixture(scope="class") def s3_models_second_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=model_namespace.name, runtime=serving_runtime_from_template.name, diff --git a/tests/model_serving/model_server/serverless/conftest.py b/tests/model_serving/model_server/serverless/conftest.py index c56c804ac..acba01860 100644 --- a/tests/model_serving/model_server/serverless/conftest.py +++ b/tests/model_serving/model_server/serverless/conftest.py @@ -38,7 +38,7 @@ def inference_service_patched_replicas( @pytest.fixture def inference_service_updated_canary_config( - request: FixtureRequest, admin_client: DynamicClient, ovms_kserve_inference_service: InferenceService + request: FixtureRequest, unprivileged_client: DynamicClient, ovms_kserve_inference_service: InferenceService ) -> Generator[InferenceService, Any, Any]: canary_percent = request.param["canary-traffic-percent"] predictor_config = { @@ -53,7 +53,7 @@ def inference_service_updated_canary_config( with ResourceEditor(patches={ovms_kserve_inference_service: predictor_config}): wait_for_canary_rollout(isvc=ovms_kserve_inference_service, percentage=canary_percent) verify_no_failed_pods( - client=admin_client, + client=unprivileged_client, isvc=ovms_kserve_inference_service, timeout=Timeout.TIMEOUT_2MIN, ) @@ -78,13 +78,13 @@ def multiple_onnx_inference_requests( @pytest.fixture(scope="class") def s3_mnist_serverless_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name="mnist-model", namespace=model_namespace.name, runtime=ovms_kserve_serving_runtime.name, diff --git a/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py b/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py index 4058fa8fc..183ac5b71 100644 --- a/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py +++ b/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py @@ -45,13 +45,13 @@ class TestConcurrencyAutoScale: @pytest.mark.dependency(name="test_auto_scale_using_concurrency") def test_auto_scale_using_concurrency( self, - admin_client, + unprivileged_client, s3_models_inference_service, multiple_onnx_inference_requests, ): """Verify model is successfully scaled up based on concurrency metrics (KPA)""" for pods in inference_service_pods_sampler( - client=admin_client, + client=unprivileged_client, isvc=s3_models_inference_service, timeout=Timeout.TIMEOUT_2MIN, sleep=10, @@ -61,10 +61,10 @@ def test_auto_scale_using_concurrency( return @pytest.mark.dependency(requires=["test_auto_scale_using_concurrency"]) - def test_pods_scaled_down_when_no_requests(self, admin_client, s3_models_inference_service): + def test_pods_scaled_down_when_no_requests(self, unprivileged_client, s3_models_inference_service): """Verify auto-scaled pods are deleted when there are no inference requests""" for pods in inference_service_pods_sampler( - client=admin_client, + client=unprivileged_client, isvc=s3_models_inference_service, timeout=Timeout.TIMEOUT_4MIN, sleep=10, diff --git a/tests/model_serving/model_server/serverless/test_scale_to_zero.py b/tests/model_serving/model_server/serverless/test_scale_to_zero.py index 10a67f322..c86526fac 100644 --- a/tests/model_serving/model_server/serverless/test_scale_to_zero.py +++ b/tests/model_serving/model_server/serverless/test_scale_to_zero.py @@ -60,9 +60,9 @@ def test_serverless_before_scale_to_zero(self, ovms_kserve_inference_service): ) @pytest.mark.order(2) @pytest.mark.dependency(name=NO_PODS_AFTER_SCALE_TEST_NAME) - def test_no_serverless_pods_after_scale_to_zero(self, admin_client, inference_service_patched_replicas): + def test_no_serverless_pods_after_scale_to_zero(self, unprivileged_client, inference_service_patched_replicas): """Verify pods are scaled to zero""" - verify_no_inference_pods(client=admin_client, isvc=inference_service_patched_replicas) + verify_no_inference_pods(client=unprivileged_client, isvc=inference_service_patched_replicas) @pytest.mark.dependency( name=INFERENCE_AFTER_SCALE_TEST_NAME, @@ -83,9 +83,9 @@ def test_serverless_inference_after_scale_to_zero(self, inference_service_patche depends=[INFERENCE_AFTER_SCALE_TEST_NAME], ) @pytest.mark.order(4) - def test_no_serverless_pods_when_no_traffic(self, admin_client, inference_service_patched_replicas): + def test_no_serverless_pods_when_no_traffic(self, unprivileged_client, inference_service_patched_replicas): """Verify pods are scaled to zero when no traffic is sent""" - verify_no_inference_pods(client=admin_client, isvc=inference_service_patched_replicas) + verify_no_inference_pods(client=unprivileged_client, isvc=inference_service_patched_replicas) @pytest.mark.parametrize( "inference_service_patched_replicas", @@ -93,10 +93,10 @@ def test_no_serverless_pods_when_no_traffic(self, admin_client, inference_servic indirect=True, ) @pytest.mark.order(5) - def test_serverless_pods_after_scale_to_one_replica(self, admin_client, inference_service_patched_replicas): + def test_serverless_pods_after_scale_to_one_replica(self, unprivileged_client, inference_service_patched_replicas): """Verify pod is running after scaling to 1 replica""" for deployment in Deployment.get( - client=admin_client, + client=unprivileged_client, namespace=inference_service_patched_replicas.namespace, ): if deployment.labels["serving.knative.dev/configurationGeneration"] == "3": diff --git a/tests/model_serving/model_server/storage/minio/conftest.py b/tests/model_serving/model_server/storage/minio/conftest.py index 8d0a1d45b..56449882f 100644 --- a/tests/model_serving/model_server/storage/minio/conftest.py +++ b/tests/model_serving/model_server/storage/minio/conftest.py @@ -16,13 +16,13 @@ @pytest.fixture(scope="class") def kserve_ovms_minio_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, minio_data_connection: Secret, ovms_kserve_serving_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=model_namespace.name, deployment_mode=request.param["deployment-mode"], @@ -38,10 +38,10 @@ def kserve_ovms_minio_inference_service( @pytest.fixture(scope="class") def minio_service_account( - admin_client: DynamicClient, minio_data_connection: Secret + unprivileged_client: DynamicClient, minio_data_connection: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( - client=admin_client, + client=unprivileged_client, namespace=minio_data_connection.namespace, name="ci-models-bucket-sa", secrets=[{"name": minio_data_connection.name}], @@ -52,14 +52,14 @@ def minio_service_account( @pytest.fixture(scope="class") def model_mesh_ovms_minio_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, minio_data_connection: Secret, minio_service_account: ServiceAccount, model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( - client=admin_client, + client=unprivileged_client, name=request.param["name"], namespace=model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, diff --git a/tests/model_serving/model_server/storage/pvc/conftest.py b/tests/model_serving/model_server/storage/pvc/conftest.py index 590fe70cc..4dcebd2e7 100644 --- a/tests/model_serving/model_server/storage/pvc/conftest.py +++ b/tests/model_serving/model_server/storage/pvc/conftest.py @@ -19,7 +19,7 @@ @pytest.fixture(scope="class") def ci_bucket_downloaded_model_data( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, model_pvc: PersistentVolumeClaim, aws_secret_access_key: str, @@ -29,7 +29,7 @@ def ci_bucket_downloaded_model_data( ci_s3_bucket_region: str, ) -> str: return download_model_data( - admin_client=admin_client, + client=unprivileged_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, model_namespace=model_namespace.name, @@ -43,20 +43,22 @@ def ci_bucket_downloaded_model_data( @pytest.fixture() -def predictor_pods_scope_function(admin_client: DynamicClient, pvc_inference_service: InferenceService) -> list[Pod]: +def predictor_pods_scope_function( + unprivileged_client: DynamicClient, pvc_inference_service: InferenceService +) -> list[Pod]: return get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=pvc_inference_service, ) @pytest.fixture(scope="class") def predictor_pods_scope_class( - admin_client: DynamicClient, + unprivileged_client: DynamicClient, pvc_inference_service: InferenceService, ) -> list[Pod]: return get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=pvc_inference_service, ) @@ -81,14 +83,14 @@ def patched_read_only_isvc( @pytest.fixture(scope="class") def pvc_inference_service( request: FixtureRequest, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, model_pvc: PersistentVolumeClaim, ci_bucket_downloaded_model_data: str, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": request.param["name"], "namespace": model_namespace.name, "runtime": serving_runtime_from_template.name, diff --git a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py index 0f40b294a..3b97727c6 100644 --- a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py +++ b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py @@ -62,10 +62,10 @@ def test_isvc_read_only_annotation_default_value(self, first_predictor_pod): ], indirect=True, ) - def test_isvc_read_only_annotation_false(self, admin_client, patched_read_only_isvc): + def test_isvc_read_only_annotation_false(self, unprivileged_client, patched_read_only_isvc): """Test that write access is allowed when the read only annotation is set to false""" new_pod = get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=patched_read_only_isvc, )[0] new_pod.execute( @@ -82,10 +82,10 @@ def test_isvc_read_only_annotation_false(self, admin_client, patched_read_only_i ], indirect=True, ) - def test_isvc_read_only_annotation_true(self, admin_client, patched_read_only_isvc): + def test_isvc_read_only_annotation_true(self, unprivileged_client, patched_read_only_isvc): """ """ new_pod = get_pods_by_isvc_label( - client=admin_client, + client=unprivileged_client, isvc=patched_read_only_isvc, )[0] with pytest.raises(ExecOnPodError): diff --git a/tests/model_serving/model_server/upgrade/conftest.py b/tests/model_serving/model_server/upgrade/conftest.py index 586944da6..1febfedda 100644 --- a/tests/model_serving/model_server/upgrade/conftest.py +++ b/tests/model_serving/model_server/upgrade/conftest.py @@ -31,11 +31,11 @@ @pytest.fixture(scope="session") def model_namespace_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, teardown_resources: bool, ) -> Generator[Namespace, Any, Any]: name = "upgrade-model-server" - ns = Namespace(client=admin_client, name=name) + ns = Namespace(client=unprivileged_client, name=name) if pytestconfig.option.post_upgrade: yield ns @@ -43,7 +43,7 @@ def model_namespace_scope_session( else: with create_ns( - admin_client=admin_client, + client=unprivileged_client, name=name, model_mesh_enabled=True, add_dashboard_label=True, @@ -55,7 +55,7 @@ def model_namespace_scope_session( @pytest.fixture(scope="session") def models_endpoint_s3_secret_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace_scope_session: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -65,7 +65,7 @@ def models_endpoint_s3_secret_scope_session( teardown_resources: bool, ) -> Generator[Secret, Any, Any]: secret_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "models-bucket-secret", "namespace": model_namespace_scope_session.name, } @@ -92,7 +92,7 @@ def models_endpoint_s3_secret_scope_session( @pytest.fixture(scope="session") def ci_endpoint_s3_secret_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace_scope_session: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -102,7 +102,7 @@ def ci_endpoint_s3_secret_scope_session( teardown_resources: bool, ) -> Generator[Secret, Any, Any]: secret_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "ci-bucket-secret", "namespace": model_namespace_scope_session.name, } @@ -129,12 +129,12 @@ def ci_endpoint_s3_secret_scope_session( @pytest.fixture(scope="session") def model_mesh_model_service_account_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[ServiceAccount, Any, Any]: sa_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "models-bucket-sa", "namespace": ci_endpoint_s3_secret_scope_session.namespace, } @@ -157,12 +157,12 @@ def model_mesh_model_service_account_scope_session( @pytest.fixture(scope="session") def openvino_serverless_serving_runtime_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "onnx-serverless", "namespace": model_namespace_scope_session.name, } @@ -193,13 +193,13 @@ def openvino_serverless_serving_runtime_scope_session( @pytest.fixture(scope="session") def ovms_serverless_inference_service_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, openvino_serverless_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": openvino_serverless_serving_runtime_scope_session.name, "namespace": openvino_serverless_serving_runtime_scope_session.namespace, } @@ -227,12 +227,12 @@ def ovms_serverless_inference_service_scope_session( @pytest.fixture(scope="session") def caikit_raw_serving_runtime_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "caikit-raw", "namespace": model_namespace_scope_session.name, } @@ -257,13 +257,13 @@ def caikit_raw_serving_runtime_scope_session( @pytest.fixture(scope="session") def caikit_raw_inference_service_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, caikit_raw_serving_runtime_scope_session: ServingRuntime, models_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": caikit_raw_serving_runtime_scope_session.name, "namespace": caikit_raw_serving_runtime_scope_session.namespace, } @@ -292,12 +292,12 @@ def caikit_raw_inference_service_scope_session( @pytest.fixture(scope="session") def s3_ovms_model_mesh_serving_runtime_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "ovms-model-mesh", "namespace": model_namespace_scope_session.name, } @@ -328,14 +328,14 @@ def s3_ovms_model_mesh_serving_runtime_scope_session( @pytest.fixture(scope="session") def openvino_model_mesh_inference_service_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, s3_ovms_model_mesh_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, model_mesh_model_service_account_scope_session: ServiceAccount, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": s3_ovms_model_mesh_serving_runtime_scope_session.name, "namespace": s3_ovms_model_mesh_serving_runtime_scope_session.namespace, } @@ -364,12 +364,12 @@ def openvino_model_mesh_inference_service_scope_session( @pytest.fixture(scope="session") def model_service_account_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[ServiceAccount, Any, Any]: sa_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": "upgrade-models-bucket-sa", "namespace": ci_endpoint_s3_secret_scope_session.namespace, } @@ -382,9 +382,7 @@ def model_service_account_scope_session( else: with ServiceAccount( - client=admin_client, - namespace=ci_endpoint_s3_secret_scope_session.namespace, - name="upgrade-models-bucket-sa", + **sa_kwargs, secrets=[{"name": ci_endpoint_s3_secret_scope_session.name}], teardown=teardown_resources, ) as sa: @@ -394,12 +392,12 @@ def model_service_account_scope_session( @pytest.fixture(scope="session") def http_view_role_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, ovms_authenticated_serverless_inference_service_scope_session: InferenceService, teardown_resources: bool, ) -> Generator[Role, Any, Any]: role_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": f"{ovms_authenticated_serverless_inference_service_scope_session.name}-view", } @@ -425,14 +423,14 @@ def http_view_role_scope_session( @pytest.fixture(scope="session") def http_role_binding_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, http_view_role_scope_session: Role, model_service_account_scope_session: ServiceAccount, ovms_authenticated_serverless_inference_service_scope_session: InferenceService, teardown_resources: bool, ) -> Generator[RoleBinding, Any, Any]: rb_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": f"{model_service_account_scope_session.name}-view", "namespace": ovms_authenticated_serverless_inference_service_scope_session.namespace, } @@ -465,13 +463,13 @@ def http_inference_token_scope_session( @pytest.fixture(scope="session") def ovms_authenticated_serverless_inference_service_scope_session( pytestconfig: pytest.Config, - admin_client: DynamicClient, + unprivileged_client: DynamicClient, openvino_serverless_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": admin_client, + "client": unprivileged_client, "name": f"{openvino_serverless_serving_runtime_scope_session.name}-auth", "namespace": openvino_serverless_serving_runtime_scope_session.namespace, } diff --git a/utilities/general.py b/utilities/general.py index b93c531dc..d0114e3a1 100644 --- a/utilities/general.py +++ b/utilities/general.py @@ -58,7 +58,7 @@ def b64_encoded_string(string_to_encode: str) -> str: def download_model_data( - admin_client: DynamicClient, + client: DynamicClient, aws_access_key_id: str, aws_secret_access_key: str, model_namespace: str, @@ -73,7 +73,7 @@ def download_model_data( Downloads the model data from the bucket to the PVC Args: - admin_client (DynamicClient): Admin client + client (DynamicClient): Admin client aws_access_key_id (str): AWS access key aws_secret_access_key (str): AWS secret key model_namespace (str): Namespace of the model @@ -105,7 +105,7 @@ def download_model_data( containers = [ { "name": "model-downloader", - "image": utilities.infra.get_kserve_storage_initialize_image(client=admin_client), + "image": utilities.infra.get_kserve_storage_initialize_image(client=client), "args": [ f"s3://{bucket_name}/{model_path}/", pvc_model_path, @@ -125,7 +125,7 @@ def download_model_data( volumes = [{"name": model_pvc_name, "persistentVolumeClaim": {"claimName": model_pvc_name}}] with Pod( - client=admin_client, + client=client, namespace=model_namespace, name="download-model-data", init_containers=init_containers, diff --git a/utilities/infra.py b/utilities/infra.py index dcac20180..ee48d4403 100644 --- a/utilities/infra.py +++ b/utilities/infra.py @@ -58,7 +58,7 @@ @contextmanager def create_ns( name: str | None = None, - admin_client: DynamicClient | None = None, + client: DynamicClient | None = None, unprivileged_client: DynamicClient | None = None, teardown: bool = True, delete_timeout: int = Timeout.TIMEOUT_4MIN, @@ -79,7 +79,7 @@ def create_ns( Args: name (str): namespace name. Can be overwritten by `request.param["name"]` - admin_client (DynamicClient): admin client. + client (DynamicClient): admin client. unprivileged_client (UnprivilegedClient): unprivileged client. teardown (bool): should run resource teardown delete_timeout (int): delete timeout. @@ -104,7 +104,7 @@ def create_ns( namespace_kwargs = { "name": name, - "client": admin_client, + "client": client, "teardown": teardown, "delete_timeout": delete_timeout, "label": labels or {}, @@ -126,7 +126,7 @@ def create_ns( yield project if teardown: - wait_for_serverless_pods_deletion(resource=project, admin_client=admin_client) + wait_for_serverless_pods_deletion(resource=project, admin_client=client) else: with Namespace(**namespace_kwargs) as ns: @@ -134,7 +134,7 @@ def create_ns( yield ns if teardown: - wait_for_serverless_pods_deletion(resource=ns, admin_client=admin_client) + wait_for_serverless_pods_deletion(resource=ns, admin_client=client) def wait_for_replicas_in_deployment(deployment: Deployment, replicas: int) -> None: From c08c96f5e6101b958e859416b8b95ee5c0b068f5 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 14:30:40 +0300 Subject: [PATCH 2/7] feat: use unprivileged_client --- .../model_server/upgrade/conftest.py | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/tests/model_serving/model_server/upgrade/conftest.py b/tests/model_serving/model_server/upgrade/conftest.py index 1febfedda..8d3fa52d2 100644 --- a/tests/model_serving/model_server/upgrade/conftest.py +++ b/tests/model_serving/model_server/upgrade/conftest.py @@ -31,11 +31,11 @@ @pytest.fixture(scope="session") def model_namespace_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, teardown_resources: bool, ) -> Generator[Namespace, Any, Any]: name = "upgrade-model-server" - ns = Namespace(client=unprivileged_client, name=name) + ns = Namespace(client=admin_client, name=name) if pytestconfig.option.post_upgrade: yield ns @@ -43,7 +43,7 @@ def model_namespace_scope_session( else: with create_ns( - client=unprivileged_client, + client=admin_client, name=name, model_mesh_enabled=True, add_dashboard_label=True, @@ -55,7 +55,7 @@ def model_namespace_scope_session( @pytest.fixture(scope="session") def models_endpoint_s3_secret_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, model_namespace_scope_session: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -65,7 +65,7 @@ def models_endpoint_s3_secret_scope_session( teardown_resources: bool, ) -> Generator[Secret, Any, Any]: secret_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "models-bucket-secret", "namespace": model_namespace_scope_session.name, } @@ -92,7 +92,7 @@ def models_endpoint_s3_secret_scope_session( @pytest.fixture(scope="session") def ci_endpoint_s3_secret_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, model_namespace_scope_session: Namespace, aws_access_key_id: str, aws_secret_access_key: str, @@ -102,7 +102,7 @@ def ci_endpoint_s3_secret_scope_session( teardown_resources: bool, ) -> Generator[Secret, Any, Any]: secret_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "ci-bucket-secret", "namespace": model_namespace_scope_session.name, } @@ -129,12 +129,12 @@ def ci_endpoint_s3_secret_scope_session( @pytest.fixture(scope="session") def model_mesh_model_service_account_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[ServiceAccount, Any, Any]: sa_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "models-bucket-sa", "namespace": ci_endpoint_s3_secret_scope_session.namespace, } @@ -157,12 +157,12 @@ def model_mesh_model_service_account_scope_session( @pytest.fixture(scope="session") def openvino_serverless_serving_runtime_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "onnx-serverless", "namespace": model_namespace_scope_session.name, } @@ -193,13 +193,13 @@ def openvino_serverless_serving_runtime_scope_session( @pytest.fixture(scope="session") def ovms_serverless_inference_service_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, openvino_serverless_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": openvino_serverless_serving_runtime_scope_session.name, "namespace": openvino_serverless_serving_runtime_scope_session.namespace, } @@ -227,12 +227,12 @@ def ovms_serverless_inference_service_scope_session( @pytest.fixture(scope="session") def caikit_raw_serving_runtime_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "caikit-raw", "namespace": model_namespace_scope_session.name, } @@ -257,13 +257,13 @@ def caikit_raw_serving_runtime_scope_session( @pytest.fixture(scope="session") def caikit_raw_inference_service_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, caikit_raw_serving_runtime_scope_session: ServingRuntime, models_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": caikit_raw_serving_runtime_scope_session.name, "namespace": caikit_raw_serving_runtime_scope_session.namespace, } @@ -292,12 +292,12 @@ def caikit_raw_inference_service_scope_session( @pytest.fixture(scope="session") def s3_ovms_model_mesh_serving_runtime_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, model_namespace_scope_session: Namespace, teardown_resources: bool, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "ovms-model-mesh", "namespace": model_namespace_scope_session.name, } @@ -328,14 +328,14 @@ def s3_ovms_model_mesh_serving_runtime_scope_session( @pytest.fixture(scope="session") def openvino_model_mesh_inference_service_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, s3_ovms_model_mesh_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, model_mesh_model_service_account_scope_session: ServiceAccount, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": s3_ovms_model_mesh_serving_runtime_scope_session.name, "namespace": s3_ovms_model_mesh_serving_runtime_scope_session.namespace, } @@ -364,12 +364,12 @@ def openvino_model_mesh_inference_service_scope_session( @pytest.fixture(scope="session") def model_service_account_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[ServiceAccount, Any, Any]: sa_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": "upgrade-models-bucket-sa", "namespace": ci_endpoint_s3_secret_scope_session.namespace, } @@ -382,7 +382,9 @@ def model_service_account_scope_session( else: with ServiceAccount( - **sa_kwargs, + client=admin_client, + namespace=ci_endpoint_s3_secret_scope_session.namespace, + name="upgrade-models-bucket-sa", secrets=[{"name": ci_endpoint_s3_secret_scope_session.name}], teardown=teardown_resources, ) as sa: @@ -392,12 +394,12 @@ def model_service_account_scope_session( @pytest.fixture(scope="session") def http_view_role_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, ovms_authenticated_serverless_inference_service_scope_session: InferenceService, teardown_resources: bool, ) -> Generator[Role, Any, Any]: role_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": f"{ovms_authenticated_serverless_inference_service_scope_session.name}-view", } @@ -423,14 +425,14 @@ def http_view_role_scope_session( @pytest.fixture(scope="session") def http_role_binding_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, http_view_role_scope_session: Role, model_service_account_scope_session: ServiceAccount, ovms_authenticated_serverless_inference_service_scope_session: InferenceService, teardown_resources: bool, ) -> Generator[RoleBinding, Any, Any]: rb_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": f"{model_service_account_scope_session.name}-view", "namespace": ovms_authenticated_serverless_inference_service_scope_session.namespace, } @@ -463,13 +465,13 @@ def http_inference_token_scope_session( @pytest.fixture(scope="session") def ovms_authenticated_serverless_inference_service_scope_session( pytestconfig: pytest.Config, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, openvino_serverless_serving_runtime_scope_session: ServingRuntime, ci_endpoint_s3_secret_scope_session: Secret, teardown_resources: bool, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { - "client": unprivileged_client, + "client": admin_client, "name": f"{openvino_serverless_serving_runtime_scope_session.name}-auth", "namespace": openvino_serverless_serving_runtime_scope_session.namespace, } From 03857cb4c511260993e10db35d91be2e84dd6cf7 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 14:44:28 +0300 Subject: [PATCH 3/7] feat: use unprivileged_client --- .../model_server/authentication/conftest.py | 26 +++++----- .../test_kserve_token_authentication_raw.py | 2 +- ..._kserve_token_authentication_serverless.py | 2 +- .../test_model_mesh_authentication.py | 3 +- .../model_server/components/conftest.py | 4 +- .../kserve_dsc_deployment_mode/conftest.py | 4 +- ...st_model_mesh_kserve_inference_co_exist.py | 4 +- ...eployment_serverless_inference_co_exist.py | 8 +-- .../components/test_custom_resources.py | 2 +- tests/model_serving/model_server/conftest.py | 52 +++++++++---------- .../test_isvc_env_vars_updates.py | 4 +- .../test_isvc_replicas_update.py | 2 +- .../metrics/test_model_metrics.py | 2 +- .../model_server/model_car/conftest.py | 4 +- .../model_server/model_car/test_oci_image.py | 2 +- .../test_model_mesh_multi_models.py | 4 +- .../test_model_mesh_multi_servers.py | 4 +- .../model_server/multi_node/conftest.py | 8 +-- .../multi_node/test_nvidia_multi_node.py | 2 +- .../ovms/kserve/test_onnx_serverless.py | 2 +- .../ovms/kserve/test_openvino_serverless.py | 2 +- .../ovms/model_mesh/test_openvino.py | 2 +- .../ovms/model_mesh/test_tensorflow.py | 3 +- .../model_server/private_endpoint/conftest.py | 10 ++-- .../test_kserve_private_endpoint.py | 2 +- .../test_bge_large_eng_caikit.py | 2 +- .../test_kserve_raw_routes_reconciliation.py | 2 +- .../routes/test_raw_deployment.py | 4 +- .../model_server/routes/test_serverless.py | 2 +- .../runtime_configuration/conftest.py | 4 +- .../test_kserve_model_priorities.py | 6 ++- .../model_server/serverless/conftest.py | 4 +- .../serverless/test_canary_rollout.py | 2 +- .../serverless/test_concurrency_auto_scale.py | 2 +- .../test_multiple_projects_in_ns.py | 2 +- ...l_controller_failing_while_deleted_isvc.py | 2 +- .../serverless/test_scale_to_zero.py | 2 +- .../serverless/test_zero_initial_scale.py | 2 +- .../model_server/storage/minio/conftest.py | 8 +-- .../storage/minio/test_minio_model_mesh.py | 2 +- .../minio/test_minio_raw_deployment.py | 2 +- .../storage/minio/test_minio_serverless.py | 2 +- .../model_server/storage/pvc/conftest.py | 8 +-- .../storage/pvc/test_kserve_pvc_rwx.py | 3 +- .../pvc/test_kserve_pvc_write_access.py | 3 +- 45 files changed, 115 insertions(+), 109 deletions(-) diff --git a/tests/model_serving/model_server/authentication/conftest.py b/tests/model_serving/model_server/authentication/conftest.py index 8804b98ee..9f5a46898 100644 --- a/tests/model_serving/model_server/authentication/conftest.py +++ b/tests/model_serving/model_server/authentication/conftest.py @@ -48,12 +48,12 @@ def grpc_model_service_account( @pytest.fixture(scope="class") def grpc_s3_caikit_serving_runtime( unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( client=unprivileged_client, name=f"{Protocols.GRPC}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING, multi_model=False, enable_http=False, @@ -65,7 +65,7 @@ def grpc_s3_caikit_serving_runtime( @pytest.fixture(scope="class") def grpc_s3_inference_service( unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, grpc_s3_caikit_serving_runtime: ServingRuntime, s3_models_storage_uri: str, models_endpoint_s3_secret: Secret, @@ -73,7 +73,7 @@ def grpc_s3_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.GRPC}-{ModelFormat.CAIKIT}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=grpc_s3_caikit_serving_runtime.name, model_format=grpc_s3_caikit_serving_runtime.instance.spec.supportedModelFormats[0].name, storage_key=models_endpoint_s3_secret.name, @@ -256,7 +256,7 @@ def grpc_inference_token(grpc_model_service_account: ServiceAccount, grpc_role_b def http_s3_caikit_serverless_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, models_endpoint_s3_secret: Secret, @@ -264,7 +264,7 @@ def http_s3_caikit_serverless_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, model_format=http_s3_caikit_tgis_serving_runtime.instance.spec.supportedModelFormats[0].name, deployment_mode=KServeDeploymentType.SERVERLESS, @@ -279,7 +279,7 @@ def http_s3_caikit_serverless_inference_service( def http_s3_caikit_raw_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, models_endpoint_s3_secret: Secret, @@ -288,7 +288,7 @@ def http_s3_caikit_raw_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, storage_key=models_endpoint_s3_secret.name, storage_path=urlparse(s3_models_storage_uri).path, @@ -305,15 +305,15 @@ def http_s3_caikit_raw_inference_service( def http_s3_caikit_raw_inference_service_2( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_caikit_tgis_serving_runtime: ServingRuntime, s3_models_storage_uri: str, model_service_account_2: ServiceAccount, -) -> InferenceService: +) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}-2", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_caikit_tgis_serving_runtime.name, storage_uri=s3_models_storage_uri, model_format=http_s3_caikit_tgis_serving_runtime.instance.spec.supportedModelFormats[0].name, @@ -329,12 +329,12 @@ def http_s3_caikit_raw_inference_service_2( def http_s3_caikit_tgis_serving_runtime( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING, multi_model=False, enable_http=True, diff --git a/tests/model_serving/model_server/authentication/test_kserve_token_authentication_raw.py b/tests/model_serving/model_server/authentication/test_kserve_token_authentication_raw.py index a65f5a500..f276fbd39 100644 --- a/tests/model_serving/model_server/authentication/test_kserve_token_authentication_raw.py +++ b/tests/model_serving/model_server/authentication/test_kserve_token_authentication_raw.py @@ -14,7 +14,7 @@ @pytest.mark.rawdeployment @pytest.mark.parametrize( - "model_namespace, s3_models_storage_uri", + "unprivileged_model_namespace, s3_models_storage_uri", [ pytest.param( {"name": "kserve-raw-token-authentication"}, diff --git a/tests/model_serving/model_server/authentication/test_kserve_token_authentication_serverless.py b/tests/model_serving/model_server/authentication/test_kserve_token_authentication_serverless.py index df6aa8cff..7a5534594 100644 --- a/tests/model_serving/model_server/authentication/test_kserve_token_authentication_serverless.py +++ b/tests/model_serving/model_server/authentication/test_kserve_token_authentication_serverless.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize( - "model_namespace, s3_models_storage_uri", + "unprivileged_model_namespace, s3_models_storage_uri", [ pytest.param( {"name": "kserve-token-authentication"}, diff --git a/tests/model_serving/model_server/authentication/test_model_mesh_authentication.py b/tests/model_serving/model_server/authentication/test_model_mesh_authentication.py index 59d8bf6ad..4bcbedc2f 100644 --- a/tests/model_serving/model_server/authentication/test_model_mesh_authentication.py +++ b/tests/model_serving/model_server/authentication/test_model_mesh_authentication.py @@ -12,7 +12,8 @@ @pytest.mark.parametrize( - "model_namespace, http_s3_ovms_model_mesh_serving_runtime, http_s3_openvino_model_mesh_inference_service", + "unprivileged_model_namespace, http_s3_ovms_model_mesh_serving_runtime, " + "http_s3_openvino_model_mesh_inference_service", [ pytest.param( {"name": "model-mesh-authentication", "modelmesh-enabled": True}, diff --git a/tests/model_serving/model_server/components/conftest.py b/tests/model_serving/model_server/components/conftest.py index bec687a1f..6566970ff 100644 --- a/tests/model_serving/model_server/components/conftest.py +++ b/tests/model_serving/model_server/components/conftest.py @@ -34,7 +34,7 @@ def managed_modelmesh_kserve_in_dsc( def invalid_s3_models_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_s3_bucket_name: str, model_service_account: ServiceAccount, @@ -42,7 +42,7 @@ def invalid_s3_models_inference_service( with create_isvc( client=unprivileged_client, name=request.param["name"], - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=serving_runtime_from_template.name, storage_uri=f"s3://{models_s3_bucket_name}/non-existing-path/", model_format=serving_runtime_from_template.instance.spec.supportedModelFormats[0].name, diff --git a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py index fbae56297..e41bdce11 100644 --- a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py +++ b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py @@ -58,14 +58,14 @@ def patched_default_deployment_mode_in_dsc( def ovms_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=request.param["name"], - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=ovms_kserve_serving_runtime.name, storage_path=request.param["model-dir"], storage_key=ci_endpoint_s3_secret.name, diff --git a/tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_inference_co_exist.py b/tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_inference_co_exist.py index e23cfae1b..f3e685f81 100644 --- a/tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_inference_co_exist.py +++ b/tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_inference_co_exist.py @@ -18,7 +18,7 @@ @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " "http_s3_openvino_model_mesh_inference_service", [ pytest.param( @@ -57,7 +57,7 @@ def test_model_mesh_openvino_created_after_serverless_in_namespace_rest_inferenc @pytest.mark.parametrize( - "model_namespace, http_s3_openvino_model_mesh_inference_service, ovms_kserve_serving_runtime, " + "unprivileged_model_namespace, http_s3_openvino_model_mesh_inference_service, ovms_kserve_serving_runtime, " "ovms_kserve_inference_service, ", [ pytest.param( diff --git a/tests/model_serving/model_server/components/raw_deployment_serverless_co_exist/test_raw_deployment_serverless_inference_co_exist.py b/tests/model_serving/model_server/components/raw_deployment_serverless_co_exist/test_raw_deployment_serverless_inference_co_exist.py index 4cbc34203..69fa37d85 100644 --- a/tests/model_serving/model_server/components/raw_deployment_serverless_co_exist/test_raw_deployment_serverless_inference_co_exist.py +++ b/tests/model_serving/model_server/components/raw_deployment_serverless_co_exist/test_raw_deployment_serverless_inference_co_exist.py @@ -33,7 +33,7 @@ @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " "serving_runtime_from_template, s3_models_inference_service", [ pytest.param( @@ -76,7 +76,7 @@ def test_raw_internal_deployment_caikit_created_after_serverless_in_namespace_re @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service, " "serving_runtime_from_template, s3_models_inference_service", [ pytest.param( @@ -119,7 +119,7 @@ def test_raw_external_deployment_caikit_created_after_serverless_in_namespace_re @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service," + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service," "ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( @@ -164,7 +164,7 @@ def test_serverless_openvino_created_after_raw_internal_deployment_caikit_ns_res @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service," + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service," "ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/components/test_custom_resources.py b/tests/model_serving/model_server/components/test_custom_resources.py index f9a83c1ea..78047978c 100644 --- a/tests/model_serving/model_server/components/test_custom_resources.py +++ b/tests/model_serving/model_server/components/test_custom_resources.py @@ -36,7 +36,7 @@ def wait_for_isvc_model_status(isvc: InferenceService, target_model_state: str, @pytest.mark.jira("RHOAIENG-10765") @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, invalid_s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, invalid_s3_models_inference_service", [ pytest.param( {"name": "non-existing-models-storage-path"}, diff --git a/tests/model_serving/model_server/conftest.py b/tests/model_serving/model_server/conftest.py index 06970aa69..60e589efe 100644 --- a/tests/model_serving/model_server/conftest.py +++ b/tests/model_serving/model_server/conftest.py @@ -62,7 +62,7 @@ def skip_if_no_deployed_openshift_serverless(admin_client: DynamicClient) -> Non @pytest.fixture(scope="class") def models_endpoint_s3_secret( unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, aws_access_key_id: str, aws_secret_access_key: str, models_s3_bucket_name: str, @@ -72,7 +72,7 @@ def models_endpoint_s3_secret( with s3_endpoint_secret( client=unprivileged_client, name="models-bucket-secret", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_s3_region=models_s3_bucket_region, @@ -100,12 +100,12 @@ def model_service_account( def serving_runtime_from_template( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { "client": unprivileged_client, "name": request.param["name"], - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "template_name": request.param["template-name"], "multi_model": request.param["multi-model"], "models_priorities": request.param.get("models-priorities"), @@ -126,14 +126,14 @@ def serving_runtime_from_template( def s3_models_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: isvc_kwargs = { "client": unprivileged_client, "name": request.param["name"], - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "runtime": serving_runtime_from_template.name, "model_format": serving_runtime_from_template.instance.spec.supportedModelFormats[0].name, "deployment_mode": request.param["deployment-mode"], @@ -161,12 +161,12 @@ def s3_models_inference_service( def model_pvc( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[PersistentVolumeClaim, Any, Any]: access_mode = "ReadWriteOnce" pvc_kwargs = { "name": "model-pvc", - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "client": unprivileged_client, "size": request.param["pvc-size"], } @@ -215,7 +215,7 @@ def skip_if_no_deployed_openshift_service_mesh(admin_client: DynamicClient) -> N def http_s3_openvino_model_mesh_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, @@ -223,7 +223,7 @@ def http_s3_openvino_model_mesh_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.OPENVINO}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, model_service_account=ci_service_account.name, storage_key=ci_endpoint_s3_secret.name, @@ -239,11 +239,11 @@ def http_s3_openvino_model_mesh_inference_service( def http_s3_ovms_model_mesh_serving_runtime( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { "client": unprivileged_client, - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "name": f"{Protocols.HTTP}-{ModelInferenceRuntime.OPENVINO_RUNTIME}", "template_name": RuntimeTemplates.OVMS_MODEL_MESH, "multi_model": True, @@ -279,11 +279,11 @@ def http_s3_ovms_model_mesh_serving_runtime( def ovms_kserve_serving_runtime( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { "client": unprivileged_client, - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "name": request.param["runtime-name"], "template_name": RuntimeTemplates.OVMS_KSERVE, "multi_model": False, @@ -311,7 +311,7 @@ def ovms_kserve_serving_runtime( @pytest.fixture(scope="class") def ci_endpoint_s3_secret( unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, aws_access_key_id: str, aws_secret_access_key: str, ci_s3_bucket_name: str, @@ -321,7 +321,7 @@ def ci_endpoint_s3_secret( with s3_endpoint_secret( client=unprivileged_client, name="ci-bucket-secret", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_s3_region=ci_s3_bucket_region, @@ -348,7 +348,7 @@ def ci_service_account( def ovms_kserve_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: @@ -356,7 +356,7 @@ def ovms_kserve_inference_service( isvc_kwargs = { "client": unprivileged_client, "name": f"{request.param['name']}-{deployment_mode.lower()}", - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "runtime": ovms_kserve_serving_runtime.name, "storage_path": request.param["model-dir"], "storage_key": ci_endpoint_s3_secret.name, @@ -390,14 +390,14 @@ def ovms_kserve_inference_service( def ovms_raw_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=f"{request.param['name']}-raw", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, external_route=True, runtime=ovms_kserve_serving_runtime.name, storage_path=request.param["model-dir"], @@ -413,7 +413,7 @@ def ovms_raw_inference_service( def http_s3_tensorflow_model_mesh_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, @@ -421,7 +421,7 @@ def http_s3_tensorflow_model_mesh_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.TENSORFLOW}", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, model_service_account=ci_service_account.name, storage_key=ci_endpoint_s3_secret.name, @@ -476,11 +476,11 @@ def user_workload_monitoring_config_map( def http_s3_ovms_external_route_model_mesh_serving_runtime( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: runtime_kwargs = { "client": unprivileged_client, - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "name": f"{Protocols.HTTP}-{ModelInferenceRuntime.OPENVINO_RUNTIME}-exposed", "template_name": RuntimeTemplates.OVMS_MODEL_MESH, "multi_model": True, @@ -505,7 +505,7 @@ def http_s3_ovms_external_route_model_mesh_serving_runtime( def http_s3_openvino_second_model_mesh_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ci_endpoint_s3_secret: Secret, ci_service_account: ServiceAccount, ) -> Generator[InferenceService, Any, Any]: @@ -514,7 +514,7 @@ def http_s3_openvino_second_model_mesh_inference_service( with create_isvc( client=unprivileged_client, name=f"{Protocols.HTTP}-{ModelFormat.OPENVINO}-2", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=runtime.name, model_service_account=ci_service_account.name, storage_key=ci_endpoint_s3_secret.name, diff --git a/tests/model_serving/model_server/inference_service_configuration/test_isvc_env_vars_updates.py b/tests/model_serving/model_server/inference_service_configuration/test_isvc_env_vars_updates.py index d77f45d64..c347f492b 100644 --- a/tests/model_serving/model_server/inference_service_configuration/test_isvc_env_vars_updates.py +++ b/tests/model_serving/model_server/inference_service_configuration/test_isvc_env_vars_updates.py @@ -29,7 +29,7 @@ @pytest.mark.rawdeployment @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "raw-env-update"}, @@ -59,7 +59,7 @@ def test_raw_remove_isvc_env_vars(self, removed_isvc_env_vars): @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-env-update"}, diff --git a/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py b/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py index cb1255307..5f865a711 100644 --- a/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py +++ b/tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py @@ -24,7 +24,7 @@ @pytest.mark.rawdeployment @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "raw-isvc-replicas"}, diff --git a/tests/model_serving/model_server/metrics/test_model_metrics.py b/tests/model_serving/model_server/metrics/test_model_metrics.py index d1d6dbc07..48230bbfb 100644 --- a/tests/model_serving/model_server/metrics/test_model_metrics.py +++ b/tests/model_serving/model_server/metrics/test_model_metrics.py @@ -25,7 +25,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "kserve-tgis-metrics"}, diff --git a/tests/model_serving/model_server/model_car/conftest.py b/tests/model_serving/model_server/model_car/conftest.py index 6d23bc905..e31fc1720 100644 --- a/tests/model_serving/model_server/model_car/conftest.py +++ b/tests/model_serving/model_server/model_car/conftest.py @@ -15,13 +15,13 @@ def model_car_serverless_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name="serverless-model-car", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=serving_runtime_from_template.name, storage_uri=request.param["storage-uri"], model_format=serving_runtime_from_template.instance.spec.supportedModelFormats[0].name, diff --git a/tests/model_serving/model_server/model_car/test_oci_image.py b/tests/model_serving/model_server/model_car/test_oci_image.py index 7b356cbf5..a46134dc4 100644 --- a/tests/model_serving/model_server/model_car/test_oci_image.py +++ b/tests/model_serving/model_server/model_car/test_oci_image.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, model_car_serverless_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, model_car_serverless_inference_service", [ pytest.param( {"name": f"{ModelFormat.OPENVINO}-model-car"}, diff --git a/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_models.py b/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_models.py index 8ceb2352a..3038cd528 100644 --- a/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_models.py +++ b/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_models.py @@ -14,8 +14,8 @@ @pytest.mark.parametrize( - "model_namespace, http_s3_ovms_model_mesh_serving_runtime, http_s3_openvino_model_mesh_inference_service, " - "http_s3_openvino_second_model_mesh_inference_service", + "unprivileged_model_namespace, http_s3_ovms_model_mesh_serving_runtime, " + "http_s3_openvino_model_mesh_inference_service, http_s3_openvino_second_model_mesh_inference_service", [ pytest.param( {"name": "model-mesh-multi-model", "modelmesh-enabled": True}, diff --git a/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_servers.py b/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_servers.py index 05619fdae..c631a21fa 100644 --- a/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_servers.py +++ b/tests/model_serving/model_server/model_mesh/test_model_mesh_multi_servers.py @@ -14,8 +14,8 @@ @pytest.mark.parametrize( - "model_namespace, http_s3_ovms_model_mesh_serving_runtime, http_s3_openvino_model_mesh_inference_service, " - "http_s3_openvino_second_model_mesh_inference_service", + "unprivileged_model_namespace, http_s3_ovms_model_mesh_serving_runtime, " + "http_s3_openvino_model_mesh_inference_service, http_s3_openvino_second_model_mesh_inference_service", [ pytest.param( {"name": "model-mesh-multi-server", "modelmesh-enabled": True}, diff --git a/tests/model_serving/model_server/multi_node/conftest.py b/tests/model_serving/model_server/multi_node/conftest.py index 7cee52489..ee9a3f248 100644 --- a/tests/model_serving/model_server/multi_node/conftest.py +++ b/tests/model_serving/model_server/multi_node/conftest.py @@ -43,7 +43,7 @@ def skip_if_no_gpu_nodes(nvidia_gpu_nodes: list[Node]) -> None: def models_bucket_downloaded_model_data( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, models_s3_bucket_name: str, model_pvc: PersistentVolumeClaim, aws_secret_access_key: str, @@ -55,7 +55,7 @@ def models_bucket_downloaded_model_data( client=unprivileged_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - model_namespace=model_namespace.name, + model_namespace=unprivileged_model_namespace.name, model_pvc_name=model_pvc.name, bucket_name=models_s3_bucket_name, aws_endpoint_url=models_s3_bucket_endpoint, @@ -68,12 +68,12 @@ def models_bucket_downloaded_model_data( def multi_node_serving_runtime( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ) -> Generator[ServingRuntime, Any, Any]: with ServingRuntimeFromTemplate( client=unprivileged_client, name="vllm-multinode-runtime", # TODO: rename servingruntime when RHOAIENG-16147 is resolved - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, template_name="vllm-multinode-runtime-template", multi_model=False, enable_http=True, diff --git a/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py b/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py index 43c6b9791..955d14295 100644 --- a/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py +++ b/tests/model_serving/model_server/multi_node/test_nvidia_multi_node.py @@ -26,7 +26,7 @@ @pytest.mark.parametrize( - "model_namespace, models_bucket_downloaded_model_data, model_pvc, multi_node_inference_service", + "unprivileged_model_namespace, models_bucket_downloaded_model_data, model_pvc, multi_node_inference_service", [ pytest.param( {"name": "gpu-multi-node"}, diff --git a/tests/model_serving/model_server/ovms/kserve/test_onnx_serverless.py b/tests/model_serving/model_server/ovms/kserve/test_onnx_serverless.py index 9ff395998..1c760abc7 100644 --- a/tests/model_serving/model_server/ovms/kserve/test_onnx_serverless.py +++ b/tests/model_serving/model_server/ovms/kserve/test_onnx_serverless.py @@ -16,7 +16,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "kserve-serverless-onnx"}, diff --git a/tests/model_serving/model_server/ovms/kserve/test_openvino_serverless.py b/tests/model_serving/model_server/ovms/kserve/test_openvino_serverless.py index 841ed8c51..6339bccf7 100644 --- a/tests/model_serving/model_server/ovms/kserve/test_openvino_serverless.py +++ b/tests/model_serving/model_server/ovms/kserve/test_openvino_serverless.py @@ -18,7 +18,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "kserve-serverless-openvino"}, diff --git a/tests/model_serving/model_server/ovms/model_mesh/test_openvino.py b/tests/model_serving/model_server/ovms/model_mesh/test_openvino.py index 5ea38c6f4..8ec08ba65 100644 --- a/tests/model_serving/model_server/ovms/model_mesh/test_openvino.py +++ b/tests/model_serving/model_server/ovms/model_mesh/test_openvino.py @@ -17,7 +17,7 @@ @pytest.mark.parametrize( - "model_namespace, http_s3_openvino_model_mesh_inference_service", + "unprivileged_model_namespace, http_s3_openvino_model_mesh_inference_service", [ pytest.param( {"name": "model-mesh-openvino", "modelmesh-enabled": True}, diff --git a/tests/model_serving/model_server/ovms/model_mesh/test_tensorflow.py b/tests/model_serving/model_server/ovms/model_mesh/test_tensorflow.py index 8c385eaa1..c7cfeab36 100644 --- a/tests/model_serving/model_server/ovms/model_mesh/test_tensorflow.py +++ b/tests/model_serving/model_server/ovms/model_mesh/test_tensorflow.py @@ -12,7 +12,8 @@ @pytest.mark.parametrize( - "model_namespace, http_s3_ovms_model_mesh_serving_runtime, http_s3_tensorflow_model_mesh_inference_service", + "unprivileged_model_namespace, http_s3_ovms_model_mesh_serving_runtime, " + "http_s3_tensorflow_model_mesh_inference_service", [ pytest.param( {"name": "model-mesh-tensorflow", "modelmesh-enabled": True}, diff --git a/tests/model_serving/model_server/private_endpoint/conftest.py b/tests/model_serving/model_server/private_endpoint/conftest.py index 9a3dee98d..1e18ace52 100644 --- a/tests/model_serving/model_server/private_endpoint/conftest.py +++ b/tests/model_serving/model_server/private_endpoint/conftest.py @@ -19,7 +19,7 @@ @pytest.fixture(scope="class") def diff_namespace(unprivileged_client: DynamicClient) -> Generator[Namespace, Any, Any]: - with create_ns(client=unprivileged_client, name="diff-namespace") as ns: + with create_ns(unprivileged_client=unprivileged_client, name="diff-namespace") as ns: yield ns @@ -45,11 +45,11 @@ def endpoint_isvc( @pytest.fixture() def endpoint_pod_with_istio_sidecar( - unprivileged_client: DynamicClient, model_namespace: Namespace + unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( client=unprivileged_client, - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, use_istio=True, pod_name="test-with-istio", ) as pod: @@ -58,11 +58,11 @@ def endpoint_pod_with_istio_sidecar( @pytest.fixture() def endpoint_pod_without_istio_sidecar( - unprivileged_client: DynamicClient, model_namespace: Namespace + unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace ) -> Generator[Pod, Any, Any]: with create_sidecar_pod( client=unprivileged_client, - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, use_istio=False, pod_name="test", ) as pod: diff --git a/tests/model_serving/model_server/private_endpoint/test_kserve_private_endpoint.py b/tests/model_serving/model_server/private_endpoint/test_kserve_private_endpoint.py index 77fb69d69..7574449fc 100644 --- a/tests/model_serving/model_server/private_endpoint/test_kserve_private_endpoint.py +++ b/tests/model_serving/model_server/private_endpoint/test_kserve_private_endpoint.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template", + "unprivileged_model_namespace, serving_runtime_from_template", [ pytest.param( {"name": "endpoint"}, diff --git a/tests/model_serving/model_server/raw_deployment/test_bge_large_eng_caikit.py b/tests/model_serving/model_server/raw_deployment/test_bge_large_eng_caikit.py index c78713bae..fae1e5075 100644 --- a/tests/model_serving/model_server/raw_deployment/test_bge_large_eng_caikit.py +++ b/tests/model_serving/model_server/raw_deployment/test_bge_large_eng_caikit.py @@ -17,7 +17,7 @@ @pytest.mark.smoke @pytest.mark.jira("RHOAIENG-11749") @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "raw-deployment-caikit-bge"}, diff --git a/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py b/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py index f3e80d46d..4927a8eef 100644 --- a/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py +++ b/tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_raw_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_raw_inference_service", [ pytest.param( {"name": "kserve-raw-route-reconciliation"}, diff --git a/tests/model_serving/model_server/routes/test_raw_deployment.py b/tests/model_serving/model_server/routes/test_raw_deployment.py index eb8dd215a..1f7143540 100644 --- a/tests/model_serving/model_server/routes/test_raw_deployment.py +++ b/tests/model_serving/model_server/routes/test_raw_deployment.py @@ -17,7 +17,7 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "raw-deployment-caikit-flan-rest"}, @@ -99,7 +99,7 @@ def test_disabled_rest_raw_deployment_exposed_route(self, patched_s3_caikit_kser @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "raw-deployment-caikit-flan-grpc"}, diff --git a/tests/model_serving/model_server/routes/test_serverless.py b/tests/model_serving/model_server/routes/test_serverless.py index bd4c7bb9e..6aee45ddd 100644 --- a/tests/model_serving/model_server/routes/test_serverless.py +++ b/tests/model_serving/model_server/routes/test_serverless.py @@ -17,7 +17,7 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "serverless-rcaikit-routes"}, diff --git a/tests/model_serving/model_server/runtime_configuration/conftest.py b/tests/model_serving/model_server/runtime_configuration/conftest.py index 7229954d3..ddf1fb85b 100644 --- a/tests/model_serving/model_server/runtime_configuration/conftest.py +++ b/tests/model_serving/model_server/runtime_configuration/conftest.py @@ -15,14 +15,14 @@ def s3_models_second_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, models_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=request.param["name"], - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=serving_runtime_from_template.name, model_format=serving_runtime_from_template.instance.spec.supportedModelFormats[0].name, deployment_mode=request.param["deployment-mode"], diff --git a/tests/model_serving/model_server/runtime_configuration/test_kserve_model_priorities.py b/tests/model_serving/model_server/runtime_configuration/test_kserve_model_priorities.py index c53852f37..d3c459d2b 100644 --- a/tests/model_serving/model_server/runtime_configuration/test_kserve_model_priorities.py +++ b/tests/model_serving/model_server/runtime_configuration/test_kserve_model_priorities.py @@ -26,7 +26,8 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service, s3_models_second_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service, " + "s3_models_second_inference_service", [ pytest.param( {"name": "serverless-model-priority"}, @@ -82,7 +83,8 @@ def test_serverless_model_priority_second_model( @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service, s3_models_second_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service, " + "s3_models_second_inference_service", [ pytest.param( {"name": "serverless-multi-priorities"}, diff --git a/tests/model_serving/model_server/serverless/conftest.py b/tests/model_serving/model_server/serverless/conftest.py index acba01860..3a0223a7d 100644 --- a/tests/model_serving/model_server/serverless/conftest.py +++ b/tests/model_serving/model_server/serverless/conftest.py @@ -79,14 +79,14 @@ def multiple_onnx_inference_requests( def s3_mnist_serverless_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, ovms_kserve_serving_runtime: ServingRuntime, ci_endpoint_s3_secret: Secret, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name="mnist-model", - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=ovms_kserve_serving_runtime.name, storage_key=ci_endpoint_s3_secret.name, storage_path=ModelStoragePath.MNIST_8_ONNX, diff --git a/tests/model_serving/model_server/serverless/test_canary_rollout.py b/tests/model_serving/model_server/serverless/test_canary_rollout.py index 7d6536a1c..f48c2c17e 100644 --- a/tests/model_serving/model_server/serverless/test_canary_rollout.py +++ b/tests/model_serving/model_server/serverless/test_canary_rollout.py @@ -19,7 +19,7 @@ @pytest.mark.polarion("ODS-2371") @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-canary-rollout"}, diff --git a/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py b/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py index 183ac5b71..bb2fdf448 100644 --- a/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py +++ b/tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py @@ -20,7 +20,7 @@ @pytest.mark.parametrize( - "model_namespace, serving_runtime_from_template, s3_models_inference_service", + "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service", [ pytest.param( {"name": "serverless-auto-scale"}, diff --git a/tests/model_serving/model_server/serverless/test_multiple_projects_in_ns.py b/tests/model_serving/model_server/serverless/test_multiple_projects_in_ns.py index 8ee4f352f..f4e09ed4f 100644 --- a/tests/model_serving/model_server/serverless/test_multiple_projects_in_ns.py +++ b/tests/model_serving/model_server/serverless/test_multiple_projects_in_ns.py @@ -14,7 +14,7 @@ @pytest.mark.polarion("ODS-2371") @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-multi-models"}, diff --git a/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py b/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py index 7d2e79bce..7f7f4b949 100644 --- a/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py +++ b/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py @@ -28,7 +28,7 @@ @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-maistra"}, diff --git a/tests/model_serving/model_server/serverless/test_scale_to_zero.py b/tests/model_serving/model_server/serverless/test_scale_to_zero.py index c86526fac..a19debc94 100644 --- a/tests/model_serving/model_server/serverless/test_scale_to_zero.py +++ b/tests/model_serving/model_server/serverless/test_scale_to_zero.py @@ -26,7 +26,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-scale-zero"}, diff --git a/tests/model_serving/model_server/serverless/test_zero_initial_scale.py b/tests/model_serving/model_server/serverless/test_zero_initial_scale.py index 5133fb187..204d4b175 100644 --- a/tests/model_serving/model_server/serverless/test_zero_initial_scale.py +++ b/tests/model_serving/model_server/serverless/test_zero_initial_scale.py @@ -24,7 +24,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", + "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service", [ pytest.param( {"name": "serverless-initial-scale-zero"}, diff --git a/tests/model_serving/model_server/storage/minio/conftest.py b/tests/model_serving/model_server/storage/minio/conftest.py index 56449882f..9c3914706 100644 --- a/tests/model_serving/model_server/storage/minio/conftest.py +++ b/tests/model_serving/model_server/storage/minio/conftest.py @@ -17,14 +17,14 @@ def kserve_ovms_minio_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, minio_data_connection: Secret, ovms_kserve_serving_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=request.param["name"], - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, deployment_mode=request.param["deployment-mode"], model_format=request.param["model-format"], runtime=ovms_kserve_serving_runtime.name, @@ -55,13 +55,13 @@ def model_mesh_ovms_minio_inference_service( unprivileged_client: DynamicClient, minio_data_connection: Secret, minio_service_account: ServiceAccount, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( client=unprivileged_client, name=request.param["name"], - namespace=model_namespace.name, + namespace=unprivileged_model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, model_service_account=minio_service_account.name, storage_key=minio_data_connection.name, diff --git a/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py b/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py index 7bd8b6937..b9c4995aa 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py @@ -19,7 +19,7 @@ @pytest.mark.parametrize( - "model_namespace, minio_pod, minio_data_connection, http_s3_ovms_model_mesh_serving_runtime, " + "unprivileged_model_namespace, minio_pod, minio_data_connection, http_s3_ovms_model_mesh_serving_runtime, " "model_mesh_ovms_minio_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py b/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py index 8ac62eebe..cf6553de1 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " + "unprivileged_model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " "kserve_ovms_minio_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/storage/minio/test_minio_serverless.py b/tests/model_serving/model_server/storage/minio/test_minio_serverless.py index 3ae6143dd..25ff7b11c 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_serverless.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_serverless.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " + "unprivileged_model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " "kserve_ovms_minio_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/storage/pvc/conftest.py b/tests/model_serving/model_server/storage/pvc/conftest.py index 4dcebd2e7..cf2e0e1dc 100644 --- a/tests/model_serving/model_server/storage/pvc/conftest.py +++ b/tests/model_serving/model_server/storage/pvc/conftest.py @@ -20,7 +20,7 @@ def ci_bucket_downloaded_model_data( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, model_pvc: PersistentVolumeClaim, aws_secret_access_key: str, aws_access_key_id: str, @@ -32,7 +32,7 @@ def ci_bucket_downloaded_model_data( client=unprivileged_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - model_namespace=model_namespace.name, + model_namespace=unprivileged_model_namespace.name, model_pvc_name=model_pvc.name, bucket_name=ci_s3_bucket_name, aws_endpoint_url=ci_s3_bucket_endpoint, @@ -84,7 +84,7 @@ def patched_read_only_isvc( def pvc_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - model_namespace: Namespace, + unprivileged_model_namespace: Namespace, serving_runtime_from_template: ServingRuntime, model_pvc: PersistentVolumeClaim, ci_bucket_downloaded_model_data: str, @@ -92,7 +92,7 @@ def pvc_inference_service( isvc_kwargs = { "client": unprivileged_client, "name": request.param["name"], - "namespace": model_namespace.name, + "namespace": unprivileged_model_namespace.name, "runtime": serving_runtime_from_template.name, "storage_uri": f"pvc://{model_pvc.name}/{ci_bucket_downloaded_model_data}", "model_format": serving_runtime_from_template.instance.spec.supportedModelFormats[0].name, diff --git a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py index 834acf67b..e0eb3e046 100644 --- a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py +++ b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py @@ -15,7 +15,8 @@ @pytest.mark.parametrize( - "model_namespace, ci_bucket_downloaded_model_data, model_pvc, serving_runtime_from_template, pvc_inference_service", + "unprivileged_model_namespace, ci_bucket_downloaded_model_data, model_pvc, serving_runtime_from_template, " + "pvc_inference_service", [ pytest.param( {"name": "pvc-rxw-access"}, diff --git a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py index 3b97727c6..91e149d64 100644 --- a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py +++ b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_write_access.py @@ -17,7 +17,8 @@ @pytest.mark.parametrize( - "model_namespace, ci_bucket_downloaded_model_data, model_pvc, serving_runtime_from_template, pvc_inference_service", + "unprivileged_model_namespace, ci_bucket_downloaded_model_data, model_pvc, serving_runtime_from_template," + "pvc_inference_service", [ pytest.param( {"name": "pvc-write-access"}, From e4964160d1da6cf9c3990cc31a9107bc2c083912 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 16:57:20 +0300 Subject: [PATCH 4/7] feat: use unprivileged_client --- .../test_kserve_dsc_default_deployment_mode.py | 4 ++-- utilities/infra.py | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py index 43934ffa0..f98457737 100644 --- a/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py +++ b/tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py @@ -28,7 +28,7 @@ @pytest.mark.serverless @pytest.mark.parametrize( - "default_deployment_mode_in_dsc, model_namespace, ovms_kserve_serving_runtime, ovms_inference_service", + "default_deployment_mode_in_dsc, unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_inference_service", [ pytest.param( {"default-deployment-mode": KServeDeploymentType.SERVERLESS}, @@ -102,7 +102,7 @@ def test_restarted_pod_is_serverless( @pytest.mark.rawdeployment @pytest.mark.parametrize( - "default_deployment_mode_in_dsc, model_namespace, ovms_kserve_serving_runtime, ovms_inference_service", + "default_deployment_mode_in_dsc, unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_inference_service", [ pytest.param( {"default-deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT}, diff --git a/utilities/infra.py b/utilities/infra.py index ee48d4403..5e97ec9a3 100644 --- a/utilities/infra.py +++ b/utilities/infra.py @@ -9,7 +9,11 @@ import pytest from _pytest.fixtures import FixtureRequest from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError, ResourceNotFoundError, ResourceNotUniqueError +from kubernetes.dynamic.exceptions import ( + NotFoundError, + ResourceNotFoundError, + ResourceNotUniqueError, +) from ocp_resources.catalog_source import CatalogSource from ocp_resources.cluster_service_version import ClusterServiceVersion from ocp_resources.config_map import ConfigMap @@ -104,7 +108,7 @@ def create_ns( namespace_kwargs = { "name": name, - "client": client, + "client": client or unprivileged_client, "teardown": teardown, "delete_timeout": delete_timeout, "label": labels or {}, @@ -123,6 +127,16 @@ def create_ns( with ProjectRequest(name=name, client=unprivileged_client, teardown=teardown): project = Project(**namespace_kwargs) project.wait_for_status(status=project.Status.ACTIVE, timeout=Timeout.TIMEOUT_2MIN) + if _labels := namespace_kwargs.get("label", {}): + # To patch the namespace, admin client is required + ns = Namespace(client=get_client(), name=name) + ResourceEditor({ + ns: { + "metadata": { + "labels": _labels, + } + } + }).update() yield project if teardown: From 219e2fdf6ecb0955c50aba0c786b8e17280f6380 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 17:03:15 +0300 Subject: [PATCH 5/7] feat: use unprivileged_client --- tests/model_serving/model_server/multi_node/conftest.py | 4 ++-- tests/model_serving/model_server/storage/pvc/conftest.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/model_serving/model_server/multi_node/conftest.py b/tests/model_serving/model_server/multi_node/conftest.py index ee9a3f248..a04db609a 100644 --- a/tests/model_serving/model_server/multi_node/conftest.py +++ b/tests/model_serving/model_server/multi_node/conftest.py @@ -42,7 +42,7 @@ def skip_if_no_gpu_nodes(nvidia_gpu_nodes: list[Node]) -> None: @pytest.fixture(scope="class") def models_bucket_downloaded_model_data( request: FixtureRequest, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, unprivileged_model_namespace: Namespace, models_s3_bucket_name: str, model_pvc: PersistentVolumeClaim, @@ -52,7 +52,7 @@ def models_bucket_downloaded_model_data( models_s3_bucket_region: str, ) -> str: return download_model_data( - client=unprivileged_client, + client=admin_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, model_namespace=unprivileged_model_namespace.name, diff --git a/tests/model_serving/model_server/storage/pvc/conftest.py b/tests/model_serving/model_server/storage/pvc/conftest.py index cf2e0e1dc..b7e4d4739 100644 --- a/tests/model_serving/model_server/storage/pvc/conftest.py +++ b/tests/model_serving/model_server/storage/pvc/conftest.py @@ -19,7 +19,7 @@ @pytest.fixture(scope="class") def ci_bucket_downloaded_model_data( request: FixtureRequest, - unprivileged_client: DynamicClient, + admin_client: DynamicClient, unprivileged_model_namespace: Namespace, model_pvc: PersistentVolumeClaim, aws_secret_access_key: str, @@ -29,7 +29,7 @@ def ci_bucket_downloaded_model_data( ci_s3_bucket_region: str, ) -> str: return download_model_data( - client=unprivileged_client, + client=admin_client, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, model_namespace=unprivileged_model_namespace.name, From 6056bf552a0958fd3bab33006c5addce47970451 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 19:18:44 +0300 Subject: [PATCH 6/7] feat: use unprivileged_client --- tests/conftest.py | 30 +++-------- .../model_server/storage/minio/conftest.py | 32 +++++++++--- .../storage/minio/test_minio_model_mesh.py | 4 +- .../minio/test_minio_raw_deployment.py | 2 +- .../storage/minio/test_minio_serverless.py | 2 +- .../storage/pvc/test_kserve_pvc_rwx.py | 2 +- utilities/minio.py | 52 +++++++++++++++++++ 7 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 utilities/minio.py diff --git a/tests/conftest.py b/tests/conftest.py index e7e1d6ffb..45f393d39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,6 @@ from utilities.data_science_cluster_utils import update_components_in_dsc from utilities.exceptions import ClusterLoginError -from utilities.general import get_s3_secret_dict from utilities.infra import ( verify_cluster_sanity, create_ns, @@ -35,14 +34,13 @@ ) from utilities.constants import ( AcceleratorType, - ApiGroups, DscComponents, Labels, MinIo, Protocols, ) from utilities.infra import update_configmap_data - +from utilities.minio import create_minio_data_connection_secret LOGGER = get_logger(name=__name__) @@ -455,29 +453,13 @@ def minio_data_connection( model_namespace: Namespace, minio_service: Service, ) -> Generator[Secret, Any, Any]: - data_dict = get_s3_secret_dict( - aws_access_key=MinIo.Credentials.ACCESS_KEY_VALUE, - aws_secret_access_key=MinIo.Credentials.SECRET_KEY_VALUE, # pragma: allowlist secret + with create_minio_data_connection_secret( + minio_service=minio_service, + model_namespace=model_namespace.name, aws_s3_bucket=request.param["bucket"], - aws_s3_endpoint=f"{Protocols.HTTP}://{minio_service.instance.spec.clusterIP}:{str(MinIo.Metadata.DEFAULT_PORT)}", # noqa: E501 - aws_s3_region="us-south", - ) - - with Secret( client=admin_client, - name="aws-connection-minio-data-connection", - namespace=model_namespace.name, - data_dict=data_dict, - label={ - Labels.OpenDataHub.DASHBOARD: "true", - Labels.OpenDataHubIo.MANAGED: "true", - }, - annotations={ - f"{ApiGroups.OPENDATAHUB_IO}/connection-type": "s3", - "openshift.io/display-name": "Minio Data Connection", - }, - ) as minio_secret: - yield minio_secret + ) as secret: + yield secret @pytest.fixture(scope="session") diff --git a/tests/model_serving/model_server/storage/minio/conftest.py b/tests/model_serving/model_server/storage/minio/conftest.py index 9c3914706..032040811 100644 --- a/tests/model_serving/model_server/storage/minio/conftest.py +++ b/tests/model_serving/model_server/storage/minio/conftest.py @@ -6,11 +6,13 @@ from ocp_resources.inference_service import InferenceService from ocp_resources.namespace import Namespace from ocp_resources.secret import Secret +from ocp_resources.service import Service from ocp_resources.service_account import ServiceAccount from ocp_resources.serving_runtime import ServingRuntime from utilities.constants import KServeDeploymentType from utilities.inference_utils import create_isvc +from utilities.minio import create_minio_data_connection_secret @pytest.fixture(scope="class") @@ -18,7 +20,7 @@ def kserve_ovms_minio_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, unprivileged_model_namespace: Namespace, - minio_data_connection: Secret, + unprivileged_minio_data_connection: Secret, ovms_kserve_serving_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: with create_isvc( @@ -28,7 +30,7 @@ def kserve_ovms_minio_inference_service( deployment_mode=request.param["deployment-mode"], model_format=request.param["model-format"], runtime=ovms_kserve_serving_runtime.name, - storage_key=minio_data_connection.name, + storage_key=unprivileged_minio_data_connection.name, storage_path=request.param["model-dir"], model_version=request.param["model-version"], external_route=request.param.get("external-route"), @@ -38,13 +40,13 @@ def kserve_ovms_minio_inference_service( @pytest.fixture(scope="class") def minio_service_account( - unprivileged_client: DynamicClient, minio_data_connection: Secret + unprivileged_client: DynamicClient, unprivileged_minio_data_connection: Secret ) -> Generator[ServiceAccount, Any, Any]: with ServiceAccount( client=unprivileged_client, - namespace=minio_data_connection.namespace, + namespace=unprivileged_minio_data_connection.namespace, name="ci-models-bucket-sa", - secrets=[{"name": minio_data_connection.name}], + secrets=[{"name": unprivileged_minio_data_connection.name}], ) as sa: yield sa @@ -53,7 +55,7 @@ def minio_service_account( def model_mesh_ovms_minio_inference_service( request: FixtureRequest, unprivileged_client: DynamicClient, - minio_data_connection: Secret, + unprivileged_minio_data_connection: Secret, minio_service_account: ServiceAccount, unprivileged_model_namespace: Namespace, http_s3_ovms_model_mesh_serving_runtime: ServingRuntime, @@ -64,10 +66,26 @@ def model_mesh_ovms_minio_inference_service( namespace=unprivileged_model_namespace.name, runtime=http_s3_ovms_model_mesh_serving_runtime.name, model_service_account=minio_service_account.name, - storage_key=minio_data_connection.name, + storage_key=unprivileged_minio_data_connection.name, storage_path=request.param["model-dir"], model_format=request.param["model-format"], deployment_mode=KServeDeploymentType.MODEL_MESH, model_version=request.param["model-version"], ) as isvc: yield isvc + + +@pytest.fixture(scope="class") +def unprivileged_minio_data_connection( + request: FixtureRequest, + unprivileged_client: DynamicClient, + unprivileged_model_namespace: Namespace, + minio_service: Service, +) -> Generator[Secret, Any, Any]: + with create_minio_data_connection_secret( + minio_service=minio_service, + model_namespace=unprivileged_model_namespace.name, + aws_s3_bucket=request.param["bucket"], + client=unprivileged_client, + ) as secret: + yield secret diff --git a/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py b/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py index b9c4995aa..4efdba02e 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_model_mesh.py @@ -19,8 +19,8 @@ @pytest.mark.parametrize( - "unprivileged_model_namespace, minio_pod, minio_data_connection, http_s3_ovms_model_mesh_serving_runtime, " - "model_mesh_ovms_minio_inference_service", + "unprivileged_model_namespace, minio_pod, unprivileged_minio_data_connection, " + "http_s3_ovms_model_mesh_serving_runtime, model_mesh_ovms_minio_inference_service", [ pytest.param( { diff --git a/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py b/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py index cf6553de1..d4e9a389d 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_raw_deployment.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "unprivileged_model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " + "unprivileged_model_namespace, minio_pod, unprivileged_minio_data_connection, ovms_kserve_serving_runtime, " "kserve_ovms_minio_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/storage/minio/test_minio_serverless.py b/tests/model_serving/model_server/storage/minio/test_minio_serverless.py index 25ff7b11c..e88a7b827 100644 --- a/tests/model_serving/model_server/storage/minio/test_minio_serverless.py +++ b/tests/model_serving/model_server/storage/minio/test_minio_serverless.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "unprivileged_model_namespace, minio_pod, minio_data_connection, ovms_kserve_serving_runtime, " + "unprivileged_model_namespace, minio_pod, unprivileged_minio_data_connection, ovms_kserve_serving_runtime, " "kserve_ovms_minio_inference_service", [ pytest.param( diff --git a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py index e0eb3e046..2e4246434 100644 --- a/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py +++ b/tests/model_serving/model_server/storage/pvc/test_kserve_pvc_rwx.py @@ -19,7 +19,7 @@ "pvc_inference_service", [ pytest.param( - {"name": "pvc-rxw-access"}, + {"name": "pvc-rwx-access"}, {"model-dir": "test-dir"}, {"access-modes": "ReadWriteMany", "storage-class-name": StorageClassName.NFS, "pvc-size": "4Gi"}, KSERVE_OVMS_SERVING_RUNTIME_PARAMS, diff --git a/utilities/minio.py b/utilities/minio.py new file mode 100644 index 000000000..e8ccb8269 --- /dev/null +++ b/utilities/minio.py @@ -0,0 +1,52 @@ +from contextlib import contextmanager +from typing import Any, Generator + +from kubernetes.dynamic import DynamicClient +from ocp_resources.secret import Secret +from ocp_resources.service import Service + +from utilities.constants import ApiGroups, Labels, MinIo, Protocols +from utilities.general import get_s3_secret_dict + + +@contextmanager +def create_minio_data_connection_secret( + minio_service: Service, + model_namespace: str, + aws_s3_bucket: str, + client: DynamicClient, +) -> Generator[Secret, Any, Any]: + """ + Create a secret for minio data connection. + + Args: + minio_service (Service): The service for minio. + model_namespace (str): The namespace where the model is stored. + aws_s3_bucket (str): The name of the bucket. + client (DynamicClient): The client to use for creating the secret. + + Yields: + Secret: The secret for minio data connection. + """ + data_dict = get_s3_secret_dict( + aws_access_key=MinIo.Credentials.ACCESS_KEY_VALUE, + aws_secret_access_key=MinIo.Credentials.SECRET_KEY_VALUE, # pragma: allowlist secret + aws_s3_bucket=aws_s3_bucket, + aws_s3_endpoint=f"{Protocols.HTTP}://{minio_service.instance.spec.clusterIP}:{str(MinIo.Metadata.DEFAULT_PORT)}", # noqa: E501 + aws_s3_region="us-south", + ) + with Secret( + client=client, + name="aws-connection-minio-data-connection", + namespace=model_namespace, + data_dict=data_dict, + label={ + Labels.OpenDataHub.DASHBOARD: "true", + Labels.OpenDataHubIo.MANAGED: "true", + }, + annotations={ + f"{ApiGroups.OPENDATAHUB_IO}/connection-type": "s3", + "openshift.io/display-name": "Minio Data Connection", + }, + ) as minio_secret: + yield minio_secret From ceb142521a36a9bf98aa05d1503c263eac06abd4 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 26 Apr 2025 19:44:40 +0300 Subject: [PATCH 7/7] fix: unpri selection --- tests/conftest.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 45f393d39..33f7c3ebc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import base64 import os import shutil +from ast import literal_eval from typing import Any, Callable, Generator import pytest @@ -231,10 +232,31 @@ def vllm_runtime_image(pytestconfig: pytest.Config) -> str | None: @pytest.fixture(scope="session") -def non_admin_user_password(admin_client: DynamicClient) -> tuple[str, str] | None: +def use_unprivileged_client(pytestconfig: pytest.Config) -> bool: + _use_unprivileged_client = py_config.get("use_unprivileged_client") + + if isinstance(_use_unprivileged_client, bool): + return _use_unprivileged_client + + elif isinstance(_use_unprivileged_client, str): + return literal_eval(_use_unprivileged_client) + + else: + raise ValueError( + "use_unprivileged_client is not defined.\n" + "Either pass with `--use-unprivileged-client` or " + "set in `use_unprivileged_client` in `tests/global_config.py`" + ) + + +@pytest.fixture(scope="session") +def non_admin_user_password(admin_client: DynamicClient, use_unprivileged_client: bool) -> tuple[str, str] | None: def _decode_split_data(_data: str) -> list[str]: return base64.b64decode(_data).decode().split(",") + if not use_unprivileged_client: + return None + if ldap_Secret := list( Secret.get( dyn_client=admin_client, @@ -267,17 +289,18 @@ def kubconfig_filepath() -> str: @pytest.fixture(scope="session") def unprivileged_client( admin_client: DynamicClient, + use_unprivileged_client: bool, kubconfig_filepath: str, non_admin_user_password: tuple[str, str], ) -> Generator[DynamicClient, Any, Any]: """ Provides none privileged API client. If non_admin_user_password is None, then it will raise. """ - if not py_config.get("use_unprivileged_client"): + if not use_unprivileged_client: LOGGER.warning("Unprivileged client is not enabled, using admin client") yield admin_client - if non_admin_user_password is None: + elif non_admin_user_password is None: raise ValueError("Unprivileged user not provisioned") else: