use oci for multi node test

Signed-off-by: Milind waykole <mwaykole@redhat.com>

Author: mwaykole

tests/model_serving/model_server/kserve/multi_node/conftest.py

@@ -9,7 +9,6 @@
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.node import Node
-from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
 from ocp_resources.pod import Pod
 from ocp_resources.resource import ResourceEditor
 from ocp_resources.secret import Secret
@@ -23,11 +22,9 @@
     get_pods_by_isvc_generation,
 )
 from utilities.constants import KServeDeploymentType, Labels, ModelCarImage, Protocols, Timeout
-from utilities.general import download_model_data
 from utilities.inference_utils import create_isvc
 from utilities.infra import (
     get_pods_by_isvc_label,
-    verify_no_failed_pods,
     wait_for_inference_deployment_replicas,
 )
 from utilities.serving_runtime import ServingRuntimeFromTemplate
@@ -46,31 +43,6 @@ def skip_if_no_gpu_nodes(nvidia_gpu_nodes: list[Node]) -> None:
         pytest.skip("Multi-node tests can only run on a Cluster with at least 2 GPU Worker nodes")
 
 
-@pytest.fixture(scope="class")
-def models_bucket_downloaded_model_data(
-    request: FixtureRequest,
-    admin_client: DynamicClient,
-    unprivileged_model_namespace: Namespace,
-    models_s3_bucket_name: str,
-    model_pvc: PersistentVolumeClaim,
-    aws_secret_access_key: str,
-    aws_access_key_id: str,
-    models_s3_bucket_endpoint: str,
-    models_s3_bucket_region: str,
-) -> str:
-    return download_model_data(
-        client=admin_client,
-        aws_access_key_id=aws_access_key_id,
-        aws_secret_access_key=aws_secret_access_key,
-        model_namespace=unprivileged_model_namespace.name,
-        model_pvc_name=model_pvc.name,
-        bucket_name=models_s3_bucket_name,
-        aws_endpoint_url=models_s3_bucket_endpoint,
-        aws_default_region=models_s3_bucket_region,
-        model_path=request.param["model-dir"],
-    )
-
-
 @pytest.fixture(scope="class")
 def multi_node_serving_runtime(
     request: FixtureRequest,
@@ -93,26 +65,47 @@ def multi_node_inference_service(
     request: FixtureRequest,
     unprivileged_client: DynamicClient,
     multi_node_serving_runtime: ServingRuntime,
-    model_pvc: PersistentVolumeClaim,
-    models_bucket_downloaded_model_data: str,
 ) -> Generator[InferenceService, Any, Any]:
+    resources = {
+        "requests": {
+            "cpu": "1",
+            "memory": "4G",
+        },
+        "limits": {
+            "cpu": "2",
+            "memory": "12G",
+        },
+    }
+
+    worker_resources = {
+        "containers": [
+            {
+                "name": "worker-container",
+                "resources": resources,
+            }
+        ]
+    }
+
     with create_isvc(
         client=unprivileged_client,
         name=request.param["name"],
         namespace=multi_node_serving_runtime.namespace,
         runtime=multi_node_serving_runtime.name,
-        storage_uri=f"pvc://{model_pvc.name}/{models_bucket_downloaded_model_data}",
+        storage_uri=ModelCarImage.GRANITE_8B_CODE_INSTRUCT,
         model_format=multi_node_serving_runtime.instance.spec.supportedModelFormats[0].name,
         deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
         autoscaler_mode="none",
-        multi_node_worker_spec={},
+        resources=resources,
+        multi_node_worker_spec=worker_resources,
         wait_for_predictor_pods=False,
+        timeout=Timeout.TIMEOUT_30MIN,
     ) as isvc:
         wait_for_inference_deployment_replicas(
             client=unprivileged_client,
             isvc=isvc,
             expected_num_deployments=2,
             runtime_name=multi_node_serving_runtime.name,
+            timeout=Timeout.TIMEOUT_15MIN,
         )
         yield isvc
 
@@ -269,16 +262,11 @@ def deleted_multi_node_pod(
         role=request.param["pod-role"],
     )
 
-    verify_no_failed_pods(
-        client=unprivileged_client,
-        isvc=multi_node_inference_service,
-        timeout=Timeout.TIMEOUT_10MIN,
-    )
-
     wait_for_inference_deployment_replicas(
         client=unprivileged_client,
         isvc=multi_node_inference_service,
         expected_num_deployments=2,
+        timeout=Timeout.TIMEOUT_15MIN,
     )
 
     _warmup_inference_and_wait_for_recovery(
@@ -317,7 +305,7 @@ def _warmup_inference_and_wait_for_recovery(
     ]
 
     for sample in TimeoutSampler(
-        wait_timeout=Timeout.TIMEOUT_10MIN,
+        wait_timeout=Timeout.TIMEOUT_30MIN,
         sleep=30,
         func=_probe_inference_health,
         client=client,

tests/model_serving/model_server/kserve/multi_node/test_nvidia_multi_node.py

@@ -1,5 +1,3 @@
-from typing import Any
-
 import pytest
 import structlog
 
@@ -14,12 +12,12 @@
     verify_ray_status,
 )
 from tests.model_serving.model_server.utils import verify_inference_response
-from utilities.constants import Labels, Protocols, StorageClassName
+from utilities.constants import Protocols
 from utilities.manifests.vllm import VLLM_INFERENCE_CONFIG
 
 pytestmark = [
     pytest.mark.rawdeployment,
-    pytest.mark.usefixtures("skip_if_no_gpu_nodes", "skip_if_no_nfs_storage_class"),
+    pytest.mark.usefixtures("skip_if_no_gpu_nodes"),
     pytest.mark.model_server_gpu,
     pytest.mark.multinode,
     pytest.mark.gpu,
@@ -31,16 +29,10 @@
 
 
 @pytest.mark.parametrize(
-    "unprivileged_model_namespace, models_bucket_downloaded_model_data, model_pvc, multi_node_inference_service",
+    "unprivileged_model_namespace, multi_node_inference_service",
     [
         pytest.param(
             {"name": "gpu-multi-node"},
-            {"model-dir": "granite-8b-code-base"},
-            {
-                "access-modes": "ReadWriteMany",
-                "storage-class-name": StorageClassName.NFS,
-                "pvc-size": "40Gi",
-            },
             {"name": "multi-vllm"},
         )
     ],
@@ -50,7 +42,7 @@ class TestMultiNode:
     """Validate multi-node GPU inference with Ray-based vLLM serving on KServe.
 
     Steps:
-        1. Deploy a Granite-8B model on a multi-node vLLM inference service backed by PVC storage.
+        1. Deploy a Granite-8B model on a multi-node vLLM inference service backed by OCI model image.
         2. Verify Ray cluster health and NVIDIA GPU status across head and worker pods.
         3. Validate default runtime worker spec (tensorParallelSize=1, pipelineParallelSize=2).
         4. Confirm pods are distributed across GPU nodes and TLS certificates are provisioned.
@@ -200,22 +192,19 @@ def test_multi_node_basic_external_inference(self, patched_multi_node_isvc_exter
         indirect=True,
     )
     def test_multi_node_tensor_parallel_size_propagation(self, unprivileged_client, patched_multi_node_spec):
-        """Test multi node tensor parallel size (number of GPUs per pod) propagation to pod config"""
-        isvc_parallel_size = str(patched_multi_node_spec.instance.spec.predictor.workerSpec.tensorParallelSize)
-
-        failed_pods: list[dict[str, Any]] = []
-
-        for pod in get_pods_by_isvc_generation(client=unprivileged_client, isvc=patched_multi_node_spec):
-            pod_resources = pod.instance.spec.containers[0].resources
-            if not (
-                isvc_parallel_size
-                == pod_resources.limits[Labels.Nvidia.NVIDIA_COM_GPU]
-                == pod_resources.requests[Labels.Nvidia.NVIDIA_COM_GPU]
-            ):
-                failed_pods.append({pod.name: pod_resources})
-
-        if failed_pods:
-            pytest.fail(f"Failed pods resources : {failed_pods}, expected tesnor parallel size {isvc_parallel_size}")
+        """Test multi node tensor parallel size propagation to pod count"""
+        worker_spec = patched_multi_node_spec.instance.spec.predictor.workerSpec
+        expected_pod_count = worker_spec.tensorParallelSize * worker_spec.pipelineParallelSize
+
+        pods = get_pods_by_isvc_generation(client=unprivileged_client, isvc=patched_multi_node_spec)
+
+        if len(pods) != expected_pod_count:
+            pytest.fail(
+                f"Expected {expected_pod_count} pods "
+                f"(tensorParallelSize={worker_spec.tensorParallelSize} "
+                f"* pipelineParallelSize={worker_spec.pipelineParallelSize}), "
+                f"but found {len(pods)} pods"
+            )
 
     @pytest.mark.parametrize(
         "patched_multi_node_spec",