feat: add llmisvcs for multinode dp+ep, singlenode p/d

Author: VedantMahabaleshwarkar

tests/model_serving/model_server/llmd/conftest.py

@@ -260,61 +260,60 @@ def llmd_inference_service_gpu(
 
 
 @pytest.fixture(scope="class")
-def deepseek_r1_inference_service(
+def llmisvc_multinode_dp_ep(
+    request: FixtureRequest,
     admin_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
 ) -> Generator[LLMInferenceService, None, None]:
-    """Fixture for DeepSeek R1 0528 model with multi-node configuration."""
-    service_name = "deepseek-r1-0528"
+    """Fixture for DeepSeek Coder V2 model with multi-node configuration optimized for GCP."""
+    # Extract parameters from pytest.mark.parametrize or use defaults
+    params = getattr(request, "param", {})
+    if not isinstance(params, dict):
+        params = {}
+
+    service_name = params.get("service_name", "deepseek-coder-v2")
+    storage_uri = params.get("storage_uri", "hf://deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct")
+    model_name = params.get("model_name", "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct")
 
     # Define common environment variables for both template and worker
     common_env = [
+        {"name": "VLLM_API_SERVER_COUNT", "value": "1"},
         {"name": "VLLM_LOGGING_LEVEL", "value": "INFO"},
-        {"name": "KSERVE_INFER_ROCE", "value": "true"},
         {"name": "CUDA_DEVICE_ORDER", "value": "PCI_BUS_ID"},
-        # Memory optimizations
         {"name": "VLLM_ADDITIONAL_ARGS", "value": "--gpu-memory-utilization 0.95 --max-model-len 8192 --enforce-eager"},
-        {"name": "VLLM_ALL2ALL_BACKEND", "value": "deepep_high_throughput"},
+        {"name": "VLLM_ALL2ALL_BACKEND", "value": "naive"},
         {"name": "PYTORCH_CUDA_ALLOC_CONF", "value": "expandable_segments:True"},
-        # Essential NCCL configuration
-        {"name": "NCCL_IB_GID_INDEX", "value": "3"},
+        {"name": "NCCL_IB_DISABLE", "value": "1"},
+        {"name": "NCCL_NET_GDR_LEVEL", "value": "0"},
+        {"name": "NCCL_P2P_LEVEL", "value": "NVL"},
+        {"name": "NCCL_SOCKET_IFNAME", "value": "eth0"},
+        {"name": "NCCL_NSOCKS_PERTHREAD", "value": "2"},
+        {"name": "NCCL_SOCKET_NTHREADS", "value": "2"},
+        {"name": "NCCL_BUFFSIZE", "value": "2097152"},
         {"name": "NCCL_DEBUG", "value": "WARN"},
-        {"name": "NCCL_SOCKET_IFNAME", "value": "net1"},
-        {"name": "NCCL_IB_TIMEOUT", "value": "100"},
-        # NVSHMEM configuration - optimized for stability
-        {"name": "NVSHMEM_REMOTE_TRANSPORT", "value": "ibgda"},
+        {"name": "NVSHMEM_REMOTE_TRANSPORT", "value": "ucx"},
+        {"name": "NVSHMEM_DISABLE_CUDA_VMM", "value": "0"},
         {"name": "NVSHMEM_BOOTSTRAP_TWO_STAGE", "value": "1"},
         {"name": "NVSHMEM_BOOTSTRAP_TIMEOUT", "value": "300"},
-        {"name": "NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME", "value": "net1"},
-        {"name": "NVSHMEM_IB_GID_INDEX", "value": "3"},
-        {"name": "NVSHMEM_USE_IBGDA", "value": "1"},
-        {"name": "NVSHMEM_ENABLE_NIC_PE_MAPPING", "value": "1"},
-        {"name": "NVSHMEM_IBGDA_SUPPORT", "value": "1"},
-        {"name": "NVSHMEM_IB_ENABLE_IBGDA", "value": "1"},
-        {"name": "NVSHMEM_IBGDA_NIC_HANDLER", "value": "gpu"},
-        {"name": "NVSHMEM_DEBUG", "value": "WARN"},
-        # UCX configuration for NVSHMEM
-        {"name": "UCX_TLS", "value": "rc,sm,self,cuda_copy,cuda_ipc"},
-        {"name": "UCX_IB_GID_INDEX", "value": "3"},
-        {"name": "UCX_RC_MLX5_TM_ENABLE", "value": "n"},
-        {"name": "UCX_UD_MLX5_RX_QUEUE_LEN", "value": "1024"},
+        {"name": "NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME", "value": "eth0"},
+        {"name": "NVSHMEM_DEBUG", "value": "INFO"},
+        {"name": "UCX_TLS", "value": "tcp,sm,self,cuda_copy,cuda_ipc"},
+        {"name": "UCX_NET_DEVICES", "value": "eth0"},
         {"name": "NVIDIA_GDRCOPY", "value": "enabled"},
     ]
 
     container_resources = {
         "limits": {
             "cpu": "128",
-            "ephemeral-storage": "800Gi",
+            "ephemeral-storage": "100Gi",
             "memory": "512Gi",
             "nvidia.com/gpu": "8",
-            "rdma/roce_gdr": "1",
         },
         "requests": {
             "cpu": "64",
-            "ephemeral-storage": "800Gi",
+            "ephemeral-storage": "100Gi",
             "memory": "256Gi",
             "nvidia.com/gpu": "8",
-            "rdma/roce_gdr": "1",
         },
     }
 
@@ -327,7 +326,7 @@ def deepseek_r1_inference_service(
     }
 
     parallelism_config = {
-        "data": 32,
+        "data": 16,
         "dataLocal": 8,
         "expert": True,
         "tensor": 1,
@@ -340,7 +339,6 @@ def deepseek_r1_inference_service(
     }
 
     worker_spec = {
-        "serviceAccountName": "hfsa",
         "containers": [
             {
                 "name": "main",
@@ -351,25 +349,191 @@ def deepseek_r1_inference_service(
     }
 
     annotations = {
-        "k8s.v1.cni.cncf.io/networks": "roce-p2",
+        "security.opendatahub.io/enable-network-policies": "false",
     }
 
     with create_llmisvc(
         client=admin_client,
         name=service_name,
         namespace=unprivileged_model_namespace.name,
-        storage_uri=ModelStorage.HF_DEEPSEEK_R1_0528,
-        model_name="deepseek-ai/DeepSeek-R1-0528",
+        storage_uri=storage_uri,
+        model_name=model_name,
         replicas=1,
         parallelism=parallelism_config,
         router_config=router_config,
         container_env=common_env,
         container_resources=container_resources,
         liveness_probe=liveness_probe,
-        service_account="hfsa",
         worker_config=worker_spec,
         annotations=annotations,
         wait=True,
         timeout=Timeout.TIMEOUT_30MIN,
     ) as llm_service:
         yield llm_service
+
+
+@pytest.fixture(scope="class")
+def llmisvc_singlenode_prefill_decode(
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
+) -> Generator[LLMInferenceService, None, None]:
+    """Fixture for single-node GPU LLMInferenceService with prefill-decode separation."""
+    # Extract parameters from pytest.mark.parametrize or use defaults
+    params = getattr(request, "param", {})
+    if not isinstance(params, dict):
+        params = {}
+
+    service_name = params.get("service_name", "qwen2-7b-instruct-pd")
+    storage_uri = params.get("storage_uri", "hf://Qwen/Qwen2.5-7B-Instruct")
+    model_name = params.get("model_name", "Qwen/Qwen2.5-7B-Instruct")
+    decode_replicas = params.get("decode_replicas", 1)
+    prefill_replicas = params.get("prefill_replicas", 2)
+
+    # Common environment variables for both prefill and decode (template)
+    common_env = [
+        # Enable RDMA for KV cache transfer
+        {"name": "KSERVE_INFER_ROCE", "value": "true"},
+        # Pod IP for KV transfer side channel
+        {
+            "name": "VLLM_NIXL_SIDE_CHANNEL_HOST",
+            "valueFrom": {"fieldRef": {"fieldPath": "status.podIP"}},
+        },
+        # Enable KV cache transfer via NixlConnector (RDMA-based)
+        {
+            "name": "VLLM_ADDITIONAL_ARGS",
+            "value": '--kv_transfer_config \'{"kv_connector":"NixlConnector","kv_role":"kv_both"}\'',
+        },
+        # UCX configuration for RDMA transport
+        {"name": "UCX_PROTO_INFO", "value": "y"},
+        {"name": "UCX_TLS", "value": "rc,sm,self,cuda_copy,cuda_ipc"},
+    ]
+
+    container_resources = {
+        "limits": {
+            "cpu": "4",
+            "memory": "32Gi",
+            "nvidia.com/gpu": "1",
+            "rdma/roce_gdr": "1",
+        },
+        "requests": {
+            "cpu": "2",
+            "memory": "16Gi",
+            "nvidia.com/gpu": "1",
+            "rdma/roce_gdr": "1",
+        },
+    }
+
+    liveness_probe = {
+        "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"},
+        "initialDelaySeconds": 120,
+        "periodSeconds": 30,
+        "timeoutSeconds": 30,
+        "failureThreshold": 5,
+    }
+
+    # Scheduler config text for prefill-decode separation
+    scheduler_config_text = """apiVersion: inference.networking.x-k8s.io/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+  - type: prefill-header-handler
+  - type: prefill-filter
+  - type: decode-filter
+  - type: max-score-picker
+  - type: queue-scorer
+    parameters:
+      hashBlockSize: 5
+      maxPrefixBlocksToMatch: 256
+      lruCapacityPerServer: 31250
+  - type: pd-profile-handler
+    parameters:
+      threshold: 0
+      hashBlockSize: 5
+schedulingProfiles:
+  - name: prefill
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 1.0
+      - pluginRef: max-score-picker
+  - name: decode
+    plugins:
+      - pluginRef: decode-filter
+      - pluginRef: queue-scorer
+        weight: 1.0
+      - pluginRef: max-score-picker
+"""
+
+    # Router config with scheduler configuration for prefill-decode separation
+    router_config = {
+        "route": {},
+        "gateway": {},
+        "scheduler": {
+            "template": {
+                "containers": [
+                    {
+                        "name": "main",
+                        "args": [
+                            "--pool-name",
+                            "{{ ChildName .ObjectMeta.Name `-inference-pool` }}",
+                            "--pool-namespace",
+                            "{{ .ObjectMeta.Namespace }}",
+                            "--zap-encoder",
+                            "json",
+                            "--grpc-port",
+                            "9002",
+                            "--grpc-health-port",
+                            "9003",
+                            "--secure-serving",
+                            "--model-server-metrics-scheme",
+                            "https",
+                            "--model-server-metrics-https-insecure-skip-verify",
+                            "--cert-path",
+                            "/etc/ssl/certs",
+                            "--config-text",
+                            scheduler_config_text,
+                        ],
+                    }
+                ]
+            }
+        },
+    }
+
+    # Prefill configuration
+    prefill_config = {
+        "replicas": prefill_replicas,
+        "template": {
+            "containers": [
+                {
+                    "name": "main",
+                    "env": common_env,
+                    "resources": container_resources,
+                    "livenessProbe": liveness_probe,
+                }
+            ]
+        },
+    }
+
+    # Annotations for RoCE network
+    annotations = {
+        # RoCE network required for KV cache transfer via RDMA
+        "k8s.v1.cni.cncf.io/networks": "roce-p2",
+    }
+
+    with create_llmisvc(
+        client=admin_client,
+        name=service_name,
+        namespace=unprivileged_model_namespace.name,
+        storage_uri=storage_uri,
+        model_name=model_name,
+        replicas=decode_replicas,
+        router_config=router_config,
+        container_env=common_env,
+        container_resources=container_resources,
+        liveness_probe=liveness_probe,
+        prefill_config=prefill_config,
+        annotations=annotations,
+        wait=True,
+        timeout=Timeout.TIMEOUT_30MIN,
+    ) as llm_service:
+        yield llm_service

tests/model_serving/model_server/llmd/test_llmd_multinode.py

@@ -0,0 +1,45 @@
+import pytest
+
+from tests.model_serving.model_server.llmd.utils import (
+    verify_gateway_status,
+    verify_llm_service_status,
+    verify_llmd_no_failed_pods,
+)
+from utilities.constants import Protocols
+from utilities.llmd_utils import verify_inference_response_llmd
+from utilities.manifests.deepseek_coder_v2_lite import DEEPSEEK_CODER_V2_INFERENCE_CONFIG
+
+pytestmark = [
+    pytest.mark.llmd_gpu,
+    pytest.mark.gpu,
+    pytest.mark.model_server_gpu,
+]
+
+
+@pytest.mark.parametrize(
+    "unprivileged_model_namespace",
+    [pytest.param({"name": "llmd-multinode-test"})],
+    indirect=True,
+)
+class TestMultiNodeLLMISVC:
+    """Multi Node LLMISVC test cases."""
+
+    def test_dp_ep(self, unprivileged_client, llmd_gateway, llmisvc_multinode_dp_ep):
+        """Test multi node llmisvc with DP + EP."""
+
+        llmisvc = llmisvc_multinode_dp_ep
+
+        assert verify_gateway_status(llmd_gateway), "Gateway should be ready"
+        assert verify_llm_service_status(llmisvc), "LLMInferenceService should be ready"
+
+        verify_inference_response_llmd(
+            llm_service=llmisvc,
+            inference_config=DEEPSEEK_CODER_V2_INFERENCE_CONFIG,
+            inference_type="completions",
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+            insecure=True,
+            model_name=llmisvc.name,
+        )
+
+        verify_llmd_no_failed_pods(client=unprivileged_client, llm_service=llmisvc)

tests/model_serving/model_server/llmd/test_llmd_singlenode_pd.py

@@ -0,0 +1,45 @@
+import pytest
+
+from tests.model_serving.model_server.llmd.utils import (
+    verify_gateway_status,
+    verify_llm_service_status,
+    verify_llmd_no_failed_pods,
+)
+from utilities.constants import Protocols
+from utilities.llmd_utils import verify_inference_response_llmd
+from utilities.manifests.deepseek_coder_v2_lite import DEEPSEEK_CODER_V2_INFERENCE_CONFIG
+
+pytestmark = [
+    pytest.mark.llmd_gpu,
+    pytest.mark.gpu,
+    pytest.mark.model_server_gpu,
+]
+
+
+@pytest.mark.parametrize(
+    "unprivileged_model_namespace",
+    [pytest.param({"name": "llmd-multinode-test"})],
+    indirect=True,
+)
+class TestSingleNodePrefillDecode:
+    """Multi Node LLMISVC test cases."""
+
+    def test_prefill_decode(self, unprivileged_client, llmd_gateway, llmisvc_singlenode_prefill_decode):
+        """Test multi node llmisvc with DP + EP."""
+
+        llmisvc = llmisvc_singlenode_prefill_decode
+
+        assert verify_gateway_status(llmd_gateway), "Gateway should be ready"
+        assert verify_llm_service_status(llmisvc), "LLMInferenceService should be ready"
+
+        verify_inference_response_llmd(
+            llm_service=llmisvc,
+            inference_config=DEEPSEEK_CODER_V2_INFERENCE_CONFIG,
+            inference_type="completions",
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+            insecure=True,
+            model_name=llmisvc.name,
+        )
+
+        verify_llmd_no_failed_pods(client=unprivileged_client, llm_service=llmisvc)