[Standup] Allow per-pod VLLM cli values. (llm-d#710)

A simple example, two decode `pods` with different `--max-model-len`

```
export LLMDBENCH_VLLM_MODELSERVICE_MULTINODE=true
export LLMDBENCH_VLLM_COMMON_MAX_MODEL_LEN=4096,,32768
```
The double-comma is used to "protect" VLLM parameters which do contain
commas, such as `--model-loader-extra-config`.

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

Author: maugustosilva

scenarios/examples/spyre.sh

@@ -30,11 +30,14 @@ export LLMDBENCH_VLLM_COMMON_EXTRA_PVC_NAME=spyre-precompiled-model
 
 #export LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME=istio
 
+#export LLMDBENCH_VLLM_MODELSERVICE_MULTINODE=true
+
 export LLMDBENCH_VLLM_COMMON_ACCELERATOR_RESOURCE=ibm.com/spyre_vf
 export LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM=4
 export LLMDBENCH_VLLM_COMMON_AFFINITY="ibm.com/spyre.product:IBM_Spyre"
 export LLMDBENCH_VLLM_COMMON_MAX_NUM_BATCHED_TOKENS=1024
 export LLMDBENCH_VLLM_COMMON_MAX_MODEL_LEN=32768
+#export LLMDBENCH_VLLM_COMMON_MAX_MODEL_LEN=4096,,32768
 export LLMDBENCH_VLLM_COMMON_MAX_NUM_SEQ=32
 export LLMDBENCH_VLLM_COMMON_MAX_NUM_BATCHED_TOKENS=1024
 export LLMDBENCH_VLLM_COMMON_CPU_NR=100
@@ -110,6 +113,7 @@ cat << EOF > $LLMDBENCH_VLLM_COMMON_EXTRA_VOLUME_MOUNTS
 - name: preprocesses
   mountPath: /setup/preprocess
 EOF
+
 export LLMDBENCH_VLLM_COMMON_EXTRA_VOLUMES=$(mktemp)
 cat << EOF > $LLMDBENCH_VLLM_COMMON_EXTRA_VOLUMES
 - name: spyre-precompiled-model
@@ -134,7 +138,7 @@ export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS=0
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_ACCELERATOR_RESOURCE=$LLMDBENCH_VLLM_COMMON_ACCELERATOR_RESOURCE
 
 # Decode parameters: 2 decode pods
-export LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS=1
+export LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS=2
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM=${LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM}
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_ACCELERATOR_RESOURCE=$LLMDBENCH_VLLM_COMMON_ACCELERATOR_RESOURCE
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_CPU_NR=$LLMDBENCH_VLLM_COMMON_CPU_NR
@@ -151,13 +155,13 @@ cat << EOF > $LLMDBENCH_VLLM_MODELSERVICE_DECODE_EXTRA_ARGS
 REPLACE_ENV_LLMDBENCH_VLLM_MODELSERVICE_DECODE_PREPROCESS; \
 /home/senuser/container-scripts/simple_vllm_serve.sh /model-cache/models/REPLACE_ENV_LLMDBENCH_DEPLOY_CURRENT_MODEL  \
 --served-model-name REPLACE_ENV_LLMDBENCH_DEPLOY_CURRENT_MODEL \
---port REPLACE_ENV_LLMDBENCH_VLLM_COMMON_METRICS_PORT \
---max-model-len REPLACE_ENV_LLMDBENCH_VLLM_COMMON_MAX_MODEL_LEN \
---tensor-parallel-size REPLACE_ENV_LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM \
---max-num-seqs REPLACE_ENV_LLMDBENCH_VLLM_COMMON_MAX_NUM_SEQ \
+--port \$VLLM_METRICS_PORT \
+--max-model-len \$VLLM_MAX_MODEL_LEN \
+--tensor-parallel-size \$VLLM_TENSOR_PARALLELISM \
+--max-num-seq \$VLLM_MAX_NUM_SEQ \
 --enable-auto-tool-choice \
 --tool-call-parser granite \
---max-num-batched-tokens REPLACE_ENV_LLMDBENCH_VLLM_COMMON_MAX_NUM_BATCHED_TOKENS \
+--max-num-batched-tokens \$VLLM_MAX_NUM_BATCHED_TOKENS \
 --enable-prefix-caching
 EOF
 

scenarios/guides/inference-scheduling.sh

@@ -36,6 +36,7 @@ export LLMDBENCH_VLLM_COMMON_PVC_MODEL_CACHE_SIZE=1Ti
 #export LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME=data-science-gateway-class
 #export LLMDBENCH_VLLM_MODELSERVICE_INFERENCEPOOL_API=inference.networking.x-k8s.io/v1alpha2
 
+#export LLMDBENCH_VLLM_MODELSERVICE_MULTINODE=true
 
 # Routing configuration (via modelservice)
 export LLMDBENCH_VLLM_MODELSERVICE_INFERENCE_MODEL=true # (default is "false")
@@ -127,7 +128,7 @@ EOF
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS=0
 
 # Decode parameters: 2 decode pods
-export LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS=1
+export LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS=2
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM=$LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_CPU_NR=$LLMDBENCH_VLLM_COMMON_CPU_NR
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_CPU_MEM=$LLMDBENCH_VLLM_COMMON_CPU_MEM

setup/env.sh

@@ -176,6 +176,7 @@ export LLMDBENCH_VLLM_COMMON_PREPROCESS=${LLMDBENCH_VLLM_COMMON_PREPROCESS:-/bin
 
 # Standalone-specific parameters
 export LLMDBENCH_VLLM_COMMON_MODEL_LOADER_EXTRA_CONFIG=${LLMDBENCH_VLLM_COMMON_MODEL_LOADER_EXTRA_CONFIG:-"{}"}
+export LLMDBENCH_VLLM_STANDALONE_INFERENCE_PORT=${LLMDBENCH_VLLM_STANDALONE_INFERENCE_PORT:-${LLMDBENCH_VLLM_COMMON_INFERENCE_PORT}}
 export LLMDBENCH_VLLM_STANDALONE_PVC_MOUNTPOINT=${LLMDBENCH_VLLM_STANDALONE_PVC_MOUNTPOINT:-/model-storage}
 export LLMDBENCH_VLLM_STANDALONE_PREPROCESS=${LLMDBENCH_VLLM_COMMON_PREPROCESS}
 export LLMDBENCH_VLLM_STANDALONE_ROUTE=${LLMDBENCH_VLLM_STANDALONE_ROUTE:-1}

setup/functions.py

@@ -1936,12 +1936,18 @@ def get_model_name_from_pod(api: pykube.HTTPClient,
     curl_command = f"curl -k --no-progress-meter {ip}"
     full_command = ["/bin/bash", "-c", f"{curl_command}"]
 
+    pull_secret_ref = None
+    if ev["vllm_common_pull_secret"] :
+        pull_secret_ref = client.V1LocalObjectReference(name=ev["vllm_common_pull_secret"])
+
     while current_attempts <= total_attempts :
         pod_name = f"testinference-pod-{get_rand_string()}"
+
         pod_manifest = client.V1Pod(
             metadata=client.V1ObjectMeta(name=pod_name, namespace=ev['vllm_common_namespace'], labels={"llm-d.ai/id": f"{pod_name}"}),
             spec=client.V1PodSpec(
                 restart_policy="Never",
+                image_pull_secrets=[pull_secret_ref],
                 containers=[
                     client.V1Container(name="model", image=image, command=full_command)
                 ],
@@ -2579,7 +2585,7 @@ def get_validation_param(ev: dict, type: str = COMMON) -> ValidationParam:
             user_accelerator_nr, tp_size, dp_size
         ),
         gpu_memory_util=float(ev[f"{prefix}_accelerator_mem_util"]),
-        max_model_len=int(ev["vllm_common_max_model_len"]),
+        max_model_len=int(ev["vllm_common_max_model_len"].split(',,')[0]),
     )
 
     return validation_param

setup/preprocess/set_llmdbench_environment.py

@@ -405,9 +405,9 @@
         if is_infiniband :
             env_file_contents.append(f"export NVSHMEM_IB_ENABLE_IBGDA=\"{is_infiniband}\"")
 
-lwswi = os.getenv("LWS_WORKER_INDEX", "0")
-dpsi = os.getenv("DP_SIZE_LOCAL", "0")
-sr = int(lwswi) * int(dpsi)
+lwswi = int(os.getenv("LWS_WORKER_INDEX", "0"))
+dpsi = int(os.getenv("DP_SIZE_LOCAL", "0"))
+sr = lwswi * dpsi
 env_file_contents.append(f"export START_RANK=\"{sr}\"")
 
 env_file_contents.append("if [[ -z $LWS_WORKER_INDEX ]]; then")
@@ -438,6 +438,21 @@
     env_file_contents.append("fi")
 
 env_file_contents.append("echo")
+
+pod_name = os.uname()[1]
+if pod_name.count("decode") :
+    pod_index=eval(pod_name.split('decode-')[-1].replace('-','+'))
+if pod_name.count("prefill") :
+    pod_index=eval(pod_name.split('prefill-')[-1].replace('-','+'))
+
+for key in dict(os.environ).keys():
+    if "VLLM_" in key:
+        value = os.environ.get(key)
+        if value.count(',,') :
+            newvalue = value.split(',,')[pod_index]
+            print(f"INFO: Variable \"{key}\" with value \"{value}\" will be re-exported with \"{newvalue}\" ({pod_index})")
+            env_file_contents.append(f"export {key}={newvalue}")
+
 env_file_contents.append("echo \"Defined NCCL environment variables\"")
 env_file_contents.append("env | grep -E \"^NCCL|^UCX|^CUDA|^OMP|^NPROC|^SMOKETEST|^NVSHMEM|START|WORLD_SIZE|RANK|^MASTER\" | sort")
 env_file_contents.append("echo")

setup/steps/06_deploy_vllm_standalone_models.py

@@ -256,8 +256,8 @@ def generate_deployment_yaml(ev, model, model_label):
           httpGet:
             path: {ev["vllm_standalone_startup_probe_path"]}
             port: {ev['vllm_common_inference_port']}
-            failureThreshold: {ev["vllm_standalone_startup_probe_failure_threshold"]}
-            initialDelaySeconds: {ev["vllm_standalone_startup_probe_initial_delay"]}
+          failureThreshold: {ev["vllm_standalone_startup_probe_failure_threshold"]}
+          initialDelaySeconds: {ev["vllm_standalone_startup_probe_initial_delay"]}
           periodSeconds: 30
           timeoutSeconds: 5
         livenessProbe:
@@ -309,10 +309,10 @@ def generate_deployment_yaml(ev, model, model_label):
         - containerPort: {ev['vllm_standalone_launcher_port']}
         startupProbe:
           httpGet:
-            path: /health
-            port: {ev['vllm_standalone_launcher_port']}
-          failureThreshold: 200
-          initialDelaySeconds: {ev.get('vllm_common_initial_delay_probe', 60)}
+            path: {ev["vllm_standalone_startup_probe_path"]}
+            port: {ev["vllm_standalone_inference_port"]}
+          failureThreshold: {ev["vllm_standalone_startup_probe_failure_threshold"]}
+          initialDelaySeconds: {ev["vllm_standalone_startup_probe_initial_delay"]}
           periodSeconds: 30
           timeoutSeconds: 5
         livenessProbe:
@@ -322,8 +322,8 @@ def generate_deployment_yaml(ev, model, model_label):
           periodSeconds: 10
         readinessProbe:
           httpGet:
-            path: /health
-            port: {ev['vllm_standalone_launcher_port']}
+            path: {ev["vllm_common_readiness_probe_path"]}
+            port: {ev["vllm_common_inference_port"]}
           failureThreshold: 3
           periodSeconds: 5
         resources:

setup/steps/10_smoketest.py

@@ -8,7 +8,6 @@
 import pykube
 import ipaddress
 
-
 # Add project root to path for imports
 current_file = Path(__file__).resolve()
 project_root = current_file.parents[1]