add pod monitor support and collect metrics data (#734)

mengmeiye · web-flow · commit b86879a0abd0 · 2026-02-27T18:57:51.000-05:00
* add pod monitor support and collect metrics data

* replace ev.get() to ev[]

* fix conflict
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -22,6 +22,9 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.
 
 RUN apt-get update; \
     apt-get install -y google-cloud-sdk-gke-gcloud-auth-plugin
+# Install kubectl for in-pod cluster operations (e.g. vLLM metrics scraping)
+RUN curl -fsSL "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+    -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl
 
 RUN echo "# /etc/rsyncd: configuration file for rsync daemon mode" > /etc/rsyncd.conf; echo -e "\
 \n\
diff --git a/build/llm-d-benchmark.sh b/build/llm-d-benchmark.sh
@@ -96,6 +96,72 @@ fi
 
 env | grep ^LLMDBENCH | grep -v BASE64 | sort
 
+# Scrape vLLM /metrics from all serving pods in the namespace.
+# Usage: scrape_vllm_metrics <phase>  (phase = "pre" or "post")
+function scrape_vllm_metrics {
+  local phase=$1
+  local namespace=${LLMDBENCH_VLLM_COMMON_NAMESPACE:-llmdbench}
+  local metrics_port=${LLMDBENCH_VLLM_COMMON_METRICS_PORT:-8200}
+  local inference_port=${LLMDBENCH_VLLM_COMMON_INFERENCE_PORT:-8000}
+  local metrics_path=${LLMDBENCH_VLLM_MONITORING_METRICS_PATH:-/metrics}
+  local metrics_dir="${LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR}/vllm_metrics"
+  local timestamp
+  timestamp=$(date --iso-8601=seconds 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%S%z")
+
+  mkdir -p "${metrics_dir}"
+  echo "Scraping vLLM ${phase} metrics (namespace=${namespace}, port=${metrics_port}, fallback_port=${inference_port})..."
+
+  # Try modelservice labels first, then standalone
+  local pod_info
+  pod_info=$(kubectl --namespace "$namespace" get pods \
+    -l llm-d.ai/inferenceServing=true \
+    --field-selector=status.phase=Running \
+    -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIP}{" "}{.metadata.labels.llm-d\.ai/role}{"\n"}{end}' 2>/dev/null || true)
+
+  if [[ -z "$pod_info" ]]; then
+    pod_info=$(kubectl --namespace "$namespace" get pods \
+      -l stood-up-via=standalone \
+      --field-selector=status.phase=Running \
+      -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIP}{" "}{"standalone"}{"\n"}{end}' 2>/dev/null || true)
+  fi
+
+  if [[ -z "$pod_info" ]]; then
+    echo "WARNING: No vLLM pods found for metrics scraping in namespace ${namespace}"
+    return 0
+  fi
+
+  echo "$pod_info" | while read -r pod_name pod_ip role; do
+    [[ -z "$pod_ip" || -z "$pod_name" ]] && continue
+    local outfile="${metrics_dir}/${phase}_${pod_name}.log"
+    echo "  Scraping ${pod_name} (${pod_ip}:${metrics_port}, role=${role})..."
+    curl -s --connect-timeout 5 --max-time 30 \
+      "http://${pod_ip}:${metrics_port}${metrics_path}" > "$outfile" 2>/dev/null
+    # If metrics port fails or returns empty, fall back to inference port (standalone vLLM serves /metrics on --port)
+    if [[ ! -s "$outfile" && "$metrics_port" != "$inference_port" ]]; then
+      echo "  Retrying ${pod_name} on inference port (${pod_ip}:${inference_port})..."
+      curl -s --connect-timeout 5 --max-time 30 \
+        "http://${pod_ip}:${inference_port}${metrics_path}" > "$outfile" 2>/dev/null || \
+        echo "  WARNING: Failed to scrape metrics from ${pod_name}"
+    fi
+  done
+
+  cat > "${metrics_dir}/${phase}_metadata.json" <<METAEOF
+{
+  "phase": "${phase}",
+  "timestamp": "${timestamp}",
+  "namespace": "${namespace}",
+  "metrics_port": ${metrics_port},
+  "metrics_path": "${metrics_path}"
+}
+METAEOF
+
+  echo "vLLM ${phase} metrics scraping complete. Files saved to ${metrics_dir}/"
+}
+
+# Scrape vLLM /metrics before benchmark run
+if [[ "${LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED:-false}" == "true" ]]; then
+  scrape_vllm_metrics "pre" || echo "WARNING: Pre-benchmark metrics scrape failed"
+fi
 
 echo "Running harness: /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}"
 counter=1
@@ -113,6 +179,11 @@ while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_LOADGEN_EC -ne 0 && "${counter}" -le
 done
 echo "Harness completed: /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}"
 
+# Scrape vLLM /metrics after benchmark run
+if [[ "${LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED:-false}" == "true" ]]; then
+  scrape_vllm_metrics "post" || echo "WARNING: Post-benchmark metrics scrape failed"
+fi
+
 if [[ -f ~/fixbashrc ]]; then
   mv -f ~/fixbashrc ~/.bashrc
 fi
diff --git a/scenarios/examples/spyre.sh b/scenarios/examples/spyre.sh
@@ -71,6 +71,8 @@ cat << EOF > $LLMDBENCH_VLLM_COMMON_ENVVARS_TO_YAML
   value: SENTIENT
 - name: FLEX_DEVICE
   value: VF
+- name: VLLM_SPYRE_PERF_METRIC_LOGGING_ENABLED
+  value: '1'
 - name: FLEX_HDMA_P2PSIZE
   value: '268435456'
 - name: FLEX_HDMA_COLLSIZE
diff --git a/setup/env.sh b/setup/env.sh
@@ -150,6 +150,13 @@ export LLMDBENCH_VLLM_COMMON_FQDN=${LLMDBENCH_VLLM_COMMON_FQDN:-".svc.cluster.lo
 export LLMDBENCH_VLLM_COMMON_TIMEOUT=${LLMDBENCH_VLLM_COMMON_TIMEOUT:-3600}
 export LLMDBENCH_VLLM_COMMON_INFERENCE_PORT=${LLMDBENCH_VLLM_COMMON_INFERENCE_PORT:-"8000"}
 export LLMDBENCH_VLLM_COMMON_METRICS_PORT=${LLMDBENCH_VLLM_COMMON_METRICS_PORT:-"8200"}
+export LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED=${LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED:-false}
+
+# vLLM Prometheus PodMonitor
+export LLMDBENCH_VLLM_MONITORING_PODMONITOR_ENABLED=${LLMDBENCH_VLLM_MONITORING_PODMONITOR_ENABLED:-false}
+export LLMDBENCH_VLLM_MONITORING_SCRAPE_INTERVAL=${LLMDBENCH_VLLM_MONITORING_SCRAPE_INTERVAL:-"30s"}
+export LLMDBENCH_VLLM_MONITORING_METRICS_PATH=${LLMDBENCH_VLLM_MONITORING_METRICS_PATH:-"/metrics"}
+
 export LLMDBENCH_VLLM_COMMON_NIXL_SIDE_CHANNEL_PORT=${LLMDBENCH_VLLM_COMMON_NIXL_SIDE_CHANNEL_PORT:-"5557"}
 export LLMDBENCH_VLLM_COMMON_UCX_TLS=${LLMDBENCH_VLLM_COMMON_UCX_TLS:-"sm,cuda_ipc,cuda_copy,tcp"}
 export LLMDBENCH_VLLM_COMMON_UCX_SOCKADDR_TLS_PRIORITY=${LLMDBENCH_VLLM_COMMON_UCX_SOCKADDR_TLS_PRIORITY:-"tcp"}
diff --git a/setup/run.sh b/setup/run.sh
@@ -58,6 +58,7 @@ function show_usage {
              -v/--verbose [print the command being executed, and result (default=$LLMDBENCH_CONTROL_VERBOSE)] \n \
              -x/--dataset [url for dataset to be replayed (default=$LLMDBENCH_RUN_DATASET_URL)] \n \
              -u/--wva [deploy model with Workload Variant Autoscaler (default=$LLMDBENCH_WVA_ENABLED)] \n \
+             -f/--monitoring [enable vLLM /metrics scraping before and after each benchmark run (default=$LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED)] \n \
              -j/--parallelism [number of harness pods to be created (default=$LLMDBENCH_HARNESS_LOAD_PARALLELISM)] \n \
              -s/--wait [time to wait until the benchmark run is complete (default=$LLMDBENCH_HARNESS_WAIT_TIMEOUT, value \"0\" means \"do not wait\"] \n \
              -g/--envvarspod [list all environment variables which should be propagated to the harness pods (default=$LLMDBENCH_HARNESS_ENVVARS_TO_YAML)] \n \
@@ -197,6 +198,9 @@ while [[ $# -gt 0 ]]; do
         -u|--wva)
         export LLMDBENCH_WVA_ENABLED=1
         ;;
+        -f|--monitoring)
+        export LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED=true
+        ;;
         -z|--skip)
         export LLMDBENCH_CLIOVERRIDE_HARNESS_SKIP_RUN=1
         ;;
diff --git a/setup/standup.sh b/setup/standup.sh
@@ -40,6 +40,7 @@ function show_usage {
             -r/--release [modelservice helm chart release name (default=$LLMDBENCH_VLLM_MODELSERVICE_RELEASE)] \n \
             -x/--dataset [url for dataset to be replayed (default=$LLMDBENCH_RUN_DATASET_URL)] \n \
             -u/--wva [deploy model with Workload Variant Autoscaler (default=$LLMDBENCH_WVA_ENABLED)] \n \
+            -f/--monitoring [enable PodMonitor for Prometheus and vLLM /metrics scraping (default=$LLMDBENCH_VLLM_MONITORING_PODMONITOR_ENABLED)] \n \
             -n/--dry-run [just print the command which would have been executed (default=$LLMDBENCH_CONTROL_DRY_RUN) ] \n \
             -v/--verbose [print the command being executed, and result (default=$LLMDBENCH_CONTROL_VERBOSE) ] \n \
             -i/--non-admin [run the setup script as a non-cluster-level admin user] \n \
@@ -152,6 +153,10 @@ while [[ $# -gt 0 ]]; do
         -u|--wva)
         export LLMDBENCH_WVA_ENABLED=1
         ;;
+        -f|--monitoring)
+        export LLMDBENCH_VLLM_MONITORING_PODMONITOR_ENABLED=true
+        export LLMDBENCH_VLLM_COMMON_METRICS_SCRAPE_ENABLED=true
+        ;;
         -n|--dry-run)
         export LLMDBENCH_CLIOVERRIDE_CONTROL_DRY_RUN=1
         ;;
diff --git a/setup/steps/06_deploy_vllm_standalone_models.py b/setup/steps/06_deploy_vllm_standalone_models.py
@@ -73,7 +73,7 @@ def main():
         yamls_dir.mkdir(parents=True, exist_ok=True)
 
         # Process each model - First pass: Deploy resources
-        model_list = ev.get("deploy_model_list", "").replace(",", " ").split()
+        model_list = ev["deploy_model_list"].replace(",", " ").split()
         for model in model_list:
             # Generate filename-safe model name
             modelfn = model.replace("/", "___")
@@ -107,6 +107,17 @@ def main():
             kubectl_service_cmd = f"{ev['control_kcmd']} apply -f {service_file}"
             llmdbench_execute_cmd(actual_cmd=kubectl_service_cmd, dry_run=ev["control_dry_run"], verbose=ev["control_verbose"], fatal=True)
 
+            # Optional PodMonitor for Prometheus scraping
+            if ev["vllm_monitoring_podmonitor_enabled"] == "true":
+                podmonitor_yaml = generate_podmonitor_yaml(ev, model, model_label)
+                podmonitor_file = yamls_dir / f"{ev['current_step']}_c_podmonitor_{modelfn}.yaml"
+                with open(podmonitor_file, 'w') as f:
+                    f.write(podmonitor_yaml)
+
+                kubectl_podmonitor_cmd = f"{ev['control_kcmd']} apply -f {podmonitor_file}"
+                llmdbench_execute_cmd(actual_cmd=kubectl_podmonitor_cmd, dry_run=ev["control_dry_run"], verbose=ev["control_verbose"], fatal=False)
+                announce(f"📊 PodMonitor for \"{model}\" created for Prometheus scraping")
+
             # Optional HTTPRoute for OpenShift
             srl = "deployment,service,pods,secrets"
             if ev["control_deploy_is_openshift"] == "1" :
@@ -169,7 +180,7 @@ def main():
         propagate_standup_parameters(ev, api)
 
     else:
-        deploy_methods = ev.get("deploy_methods", "")
+        deploy_methods = ev["deploy_methods"]
         announce(f"⏭️  Environment types are \"{deploy_methods}\". Skipping this step.")
 
     return 0
@@ -254,11 +265,12 @@ def generate_deployment_yaml(ev, model, model_label):
         - name: HUGGING_FACE_HUB_TOKEN
           valueFrom:
             secretKeyRef:
-              name: {ev.get('vllm_common_hf_token_name', '')}
+              name: {ev['vllm_common_hf_token_name']}
               key: HF_TOKEN
 {additional_env}
         ports:
         - containerPort: {ev['vllm_common_inference_port']}
+          name: metrics
         startupProbe:
           httpGet:
             path: {ev["vllm_standalone_startup_probe_path"]}
@@ -309,7 +321,7 @@ def generate_deployment_yaml(ev, model, model_label):
         - name: HUGGING_FACE_HUB_TOKEN
           valueFrom:
             secretKeyRef:
-              name: {ev.get('vllm_common_hf_token_name', '')}
+              name: {ev['vllm_common_hf_token_name']}
               key: HF_TOKEN
 {additional_env}
         ports:
@@ -382,11 +394,34 @@ def generate_service_yaml(ev, model, model_label):
 """
     return service_yaml
 
+def generate_podmonitor_yaml(ev, model, model_label):
+    """Generate Kubernetes PodMonitor YAML for Prometheus to scrape vLLM standalone model metrics."""
+
+    podmonitor_yaml = f"""apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: vllm-standalone-{model_label}
+  namespace: {ev['vllm_common_namespace']}
+  labels:
+    stood-up-by: "{ev['control_username']}"
+    stood-up-from: llm-d-benchmark
+    stood-up-via: "{ev['deploy_methods']}"
+spec:
+  selector:
+    matchLabels:
+      app: vllm-standalone-{model_label}
+  podMetricsEndpoints:
+  - port: metrics
+    path: {ev['vllm_monitoring_metrics_path']}
+    interval: {ev['vllm_monitoring_scrape_interval']}
+"""
+    return podmonitor_yaml
+
 def generate_httproute_yaml(ev, model, model_label):
     """Generate HTTPRoute YAML for vLLM standalone model."""
 
     # Extract cluster URL for hostname
-    cluster_url = ev.get("cluster_url", "").replace("https://api.", "")
+    cluster_url = ev["cluster_url"].replace("https://api.", "")
 
     # Get model attributes for backend reference
     model_parameters = model_attribute(model, "parameters")
diff --git a/setup/steps/09_deploy_via_modelservice.py b/setup/steps/09_deploy_via_modelservice.py
@@ -244,6 +244,41 @@ def generate_ms_values_yaml(
 
     return clear_string(yaml_content)
 
+def generate_podmonitor_yaml(ev: dict) -> str:
+    """Generate a PodMonitor CRD for Prometheus to scrape vLLM model serving pods.
+
+    Args:
+        ev: Environment variables dictionary
+
+    Returns:
+        PodMonitor YAML manifest as string
+    """
+    model_id_label = ev["deploy_current_model_id_label"]
+    namespace = ev["vllm_common_namespace"]
+    scrape_interval = ev["vllm_monitoring_scrape_interval"]
+    metrics_path = ev["vllm_monitoring_metrics_path"]
+    metrics_port = ev["vllm_common_metrics_port"]
+
+    return f"""apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: vllm-{model_id_label}
+  namespace: {namespace}
+  labels:
+    stood-up-by: "{ev['control_username']}"
+    stood-up-from: llm-d-benchmark
+    stood-up-via: "{ev['deploy_methods']}"
+spec:
+  selector:
+    matchLabels:
+      llm-d.ai/inferenceServing: "true"
+      llm-d.ai/model: {model_id_label}
+  podMetricsEndpoints:
+  - port: "{metrics_port}"
+    path: {metrics_path}
+    interval: {scrape_interval}
+"""
+
 def define_httproute(
     ev: dict,
     single_model: bool = True
@@ -260,9 +295,9 @@ def define_httproute(
         YAML manifest for HTTPRoute
 """
     release = ev["vllm_modelservice_release"]
-    namespace = ev.get("vllm_common_namespace", "")
-    model_id_label = ev.get("deploy_current_model_id_label", "")
-    service_port = ev.get("vllm_common_inference_port", "8000")
+    namespace = ev["vllm_common_namespace"]
+    model_id_label = ev["deploy_current_model_id_label"]
+    service_port = ev["vllm_common_inference_port"]
 
     manifest=f"""apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
@@ -395,7 +430,7 @@ def main():
       # Create directory structure (Do not use "llmdbench_execute_cmd" for these commands)
       model_num = f"{model_number:02d}"
       release = ev["vllm_modelservice_release"]
-      work_dir = Path(ev.get("control_work_dir", ""))
+      work_dir = Path(ev["control_work_dir"])
       helm_dir = work_dir / "setup" / "helm" / release / model_num
 
       # Always create directory structure (even in dry-run)
@@ -491,6 +526,15 @@ def main():
       if result != 0:
           return result
 
+      # Optional PodMonitor for Prometheus scraping of vLLM pods
+      if ev["vllm_monitoring_podmonitor_enabled"] == "true":
+          podmonitor_yaml = generate_podmonitor_yaml(ev)
+          podmonitor_file = work_dir / "setup" / "yamls" / f"{ev['current_step_nr']}_podmonitor_{ev['deploy_current_model_id_label']}.yaml"
+          podmonitor_file.parent.mkdir(parents=True, exist_ok=True)
+          podmonitor_file.write_text(podmonitor_yaml)
+          kubectl_apply(api=api, manifest_data=podmonitor_yaml, dry_run=ev["control_dry_run"])
+          announce(f"📊 PodMonitor for \"{model}\" created for Prometheus scraping")
+
       # Collect decode logs
       collect_logs(ev, ev["vllm_modelservice_decode_replicas"], "decode")
 
diff --git a/setup/teardown.sh b/setup/teardown.sh
@@ -246,6 +246,7 @@ else
   hpa
   va
   servicemonitor
+  podmonitor
   pod
   pvc
 )

Original file line number	Diff line number	Diff line change
`@@ -246,6 +246,7 @@ else`
`246`	`246`	`hpa`
`247`	`247`	`va`
`248`	`248`	`servicemonitor`
	`249`	`+ podmonitor`
`249`	`250`	`pod`
`250`	`251`	`pvc`
`251`	`252`	`)`