feat: support real vLLM serving on managed GPU clusters

MicheleCampi · MicheleCampi · commit 785f762b5529 · 2026-06-13T22:51:19.000Z
build_deployment now emits the serving invocation (command [vllm, serve] +
model positional + --host/--port, --enforce-eager from warmupStrategy, plus
spec.extraArgs for engine tuning) instead of the unread VLLM_MODEL env, backs
/dev/shm with a memory emptyDir, and takes runtimeClassName from the spec
(default None) rather than hardcoding nvidia. The v0.1.0 controller was only
ever exercised against the CI pause placeholder on kind/K3s; these changes are
what let it drive a real vLLM pod on a managed cluster (GKE/EKS/AKS), where the
GPU device plugin uses the default runtime and ExtendedResourceToleration
injects the GPU toleration.

Adds spec fields runtimeClassName (Option&lt;String&gt;) and extraArgs (Vec&lt;String&gt;),
regenerates the CRD, and pins the example image to vllm/vllm-openai:v0.11.0.
diff --git a/chart/Chart.yaml b/chart/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: vllm-coldstart-operator
 description: Kubernetes operator that treats LLM cold-start as a first-class lifecycle signal for vLLM inference services.
 type: application
-version: 0.1.0
-appVersion: "latest"
+version: 0.2.0
+appVersion: "0.2.0"
diff --git a/chart/crds/crd.yaml b/chart/crds/crd.yaml
@@ -1,13 +1,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
-  annotations:
-    argocd.argoproj.io/sync-wave: "-2"
-    argocd.argoproj.io/sync-options: Prune=false
   name: vllmservices.inference.michelecampi.dev
 spec:
-  conversion:
-    strategy: None
   group: inference.michelecampi.dev
   names:
     categories: []
@@ -33,6 +28,16 @@ spec:
               treats time-to-first-token as a first-class concern, configuring the
               pod for the cold-start/throughput trade-off the spec asks for.
             properties:
+              extraArgs:
+                default: []
+                description: |-
+                  Extra command-line arguments appended to `vllm serve <model>`, e.g.
+                  ["--max-model-len", "8192", "--gpu-memory-utilization", "0.90"].
+                  Keeps engine tuning (context length, memory fraction, quantization)
+                  in the resource spec rather than baked into the operator binary.
+                items:
+                  type: string
+                type: array
               gpu:
                 default: 1
                 description: |-
@@ -61,6 +66,16 @@ spec:
                 description: Number of replicas to run.
                 format: int32
                 type: integer
+              runtimeClassName:
+                description: |-
+                  RuntimeClass for the workload pod. Cluster-dependent: K3s GPU nodes
+                  need "nvidia" to route the pod to the NVIDIA container runtime, while
+                  managed clusters (GKE, EKS, AKS) expose GPUs through the device plugin
+                  with the default runtime and define no such RuntimeClass. Leave unset
+                  (the default) on managed clusters; setting a non-existent RuntimeClass
+                  makes the API server reject the pod.
+                nullable: true
+                type: string
               warmupStrategy:
                 default: Eager
                 description: |-
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -4,10 +4,10 @@
 #    via git write-back. When spec is set, the image helper uses it verbatim.
 image:
   repository: ghcr.io/michelecampi/vllm-coldstart-operator
-  tag: 0.1.0
+  tag: 0.2.0
   pullPolicy: IfNotPresent
   # -- Managed by ArgoCD Image Updater. Full image reference, takes precedence.
-  spec: ghcr.io/michelecampi/vllm-coldstart-operator:latest@sha256:d7ccf8580f42a87ea401f13eed9188fc8892b07bd2212357e87c7919f71fb98d
+  spec: ""
 # -- Replicas for the operator Deployment itself (leader election not
 #    implemented, so keep at 1).
 replicaCount: 1
@@ -44,15 +44,18 @@ example:
     image: "registry.k8s.io/pause:3.10"
     gpu: 0
     healthPath: ""
-    # Real GPU deployment override (requires NVIDIA device plugin):
+    # Real GPU deployment override (managed cluster, e.g. GKE/EKS/AKS):
     #   name: qwen-7b
     #   spec:
     #     model: "Qwen/Qwen2.5-7B-Instruct"
     #     replicas: 1
-    #     warmupStrategy: Eager
-    #     image: "vllm/vllm-openai:latest"
+    #     warmupStrategy: Eager          # operator appends --enforce-eager
+    #     image: "vllm/vllm-openai:v0.11.0"
     #     gpu: 1
     #     healthPath: "/health"
+    #     # runtimeClassName: leave unset on managed clusters; set to "nvidia"
+    #     #   only on K3s GPU nodes that route GPUs via the nvidia runtime.
+    #     extraArgs: ["--max-model-len", "8192", "--gpu-memory-utilization", "0.90"]
 # -- Value of RUST_LOG for the operator (tracing EnvFilter). Matches the
 #    binary's built-in default.
 logLevel: "info,kube=warn"
diff --git a/src/lib.rs b/src/lib.rs
@@ -45,6 +45,20 @@ pub struct VllmServiceSpec {
     /// placeholder images that expose no HTTP health endpoint.
     #[serde(default = "default_health_path")]
     pub health_path: String,
+    /// RuntimeClass for the workload pod. Cluster-dependent: K3s GPU nodes
+    /// need "nvidia" to route the pod to the NVIDIA container runtime, while
+    /// managed clusters (GKE, EKS, AKS) expose GPUs through the device plugin
+    /// with the default runtime and define no such RuntimeClass. Leave unset
+    /// (the default) on managed clusters; setting a non-existent RuntimeClass
+    /// makes the API server reject the pod.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub runtime_class_name: Option<String>,
+    /// Extra command-line arguments appended to `vllm serve <model>`, e.g.
+    /// ["--max-model-len", "8192", "--gpu-memory-utilization", "0.90"].
+    /// Keeps engine tuning (context length, memory fraction, quantization)
+    /// in the resource spec rather than baked into the operator binary.
+    #[serde(default)]
+    pub extra_args: Vec<String>,
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default, PartialEq)]
diff --git a/src/main.rs b/src/main.rs
@@ -5,8 +5,8 @@ use std::time::Duration;
 use futures::StreamExt;
 use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec};
 use k8s_openapi::api::core::v1::{
-    Container, ContainerPort, EnvVar, HTTPGetAction, PodSpec, PodTemplateSpec, Probe,
-    ResourceRequirements,
+    Container, ContainerPort, EmptyDirVolumeSource, HTTPGetAction, PodSpec, PodTemplateSpec, Probe,
+    ResourceRequirements, Volume, VolumeMount,
 };
 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
@@ -65,9 +65,15 @@ fn build_deployment(svc: &VllmService) -> Result<Deployment, Error> {
 
     let enforce_eager = matches!(svc.spec.warmup_strategy, WarmupStrategy::Eager);
 
+    // A request for GPUs is the signal that this is a real vLLM workload
+    // rather than the inert CI placeholder (gpu=0, pause image). Only then
+    // do we inject the serving command, the GPU limit, the shared-memory
+    // volume and any tuning args; the placeholder stays a do-nothing pod.
+    let serving = svc.spec.gpu > 0;
+
     // GPU resource limit, only when requested. With gpu=0 (CI / CPU-only)
     // the container requests no GPU and schedules on any node.
-    let resources = if svc.spec.gpu > 0 {
+    let resources = if serving {
         let mut limits = BTreeMap::new();
         limits.insert(
             "nvidia.com/gpu".to_string(),
@@ -81,6 +87,54 @@ fn build_deployment(svc: &VllmService) -> Result<Deployment, Error> {
         None
     };
 
+    // Invocation. The vllm/vllm-openai image's entrypoint is `vllm serve`,
+    // but we set command+args explicitly so the operator does not depend on
+    // the image's entrypoint staying stable across tags. `vllm serve` takes
+    // the model as a positional argument; warmupStrategy maps to
+    // --enforce-eager (CUDA graphs off => faster cold start); extraArgs
+    // carries per-deployment engine tuning (e.g. --max-model-len,
+    // --gpu-memory-utilization). The placeholder keeps the pause entrypoint.
+    let (command, args) = if serving {
+        let mut a = vec![
+            svc.spec.model.clone(),
+            "--host".to_string(),
+            "0.0.0.0".to_string(),
+            "--port".to_string(),
+            "8000".to_string(),
+        ];
+        if enforce_eager {
+            a.push("--enforce-eager".to_string());
+        }
+        a.extend(svc.spec.extra_args.iter().cloned());
+        (Some(vec!["vllm".to_string(), "serve".to_string()]), Some(a))
+    } else {
+        (None, None)
+    };
+
+    // vLLM uses /dev/shm for intra-process tensor transfer; the container
+    // runtime's default 64MiB is too small and causes hard-to-diagnose
+    // crashes under load. Back it with a memory-medium emptyDir on real
+    // serving pods. The placeholder needs none.
+    let (volumes, volume_mounts) = if serving {
+        (
+            Some(vec![Volume {
+                name: "dshm".to_string(),
+                empty_dir: Some(EmptyDirVolumeSource {
+                    medium: Some("Memory".to_string()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }]),
+            Some(vec![VolumeMount {
+                name: "dshm".to_string(),
+                mount_path: "/dev/shm".to_string(),
+                ..Default::default()
+            }]),
+        )
+    } else {
+        (None, None)
+    };
+
     // Readiness probe, built only when health_path is non-empty. This is
     // what makes "Ready" mean "warm": when set, Kubernetes marks the pod
     // ready only once the server answers on this endpoint, so the
@@ -104,45 +158,33 @@ fn build_deployment(svc: &VllmService) -> Result<Deployment, Error> {
     let container = Container {
         name: "inference".to_string(),
         image: Some(svc.spec.image.clone()),
+        command,
+        args,
         ports: Some(vec![ContainerPort {
             container_port: 8000,
             name: Some("http".to_string()),
             ..Default::default()
         }]),
-        env: Some(vec![
-            EnvVar {
-                name: "VLLM_MODEL".to_string(),
-                value: Some(svc.spec.model.clone()),
-                ..Default::default()
-            },
-            EnvVar {
-                name: "VLLM_ENFORCE_EAGER".to_string(),
-                value: Some(enforce_eager.to_string()),
-                ..Default::default()
-            },
-        ]),
         resources,
         readiness_probe,
+        volume_mounts,
         ..Default::default()
     };
 
-    // On K3s the default runtime is runc; a pod needs the "nvidia"
-    // RuntimeClass to actually get GPU access. Request it only when GPUs
-    // are requested, so gpu=0 (CI / CPU) pods schedule with the default
-    // runtime and need no RuntimeClass installed on the cluster.
-    let runtime_class_name = if svc.spec.gpu > 0 {
-        Some("nvidia".to_string())
-    } else {
-        None
-    };
+    // RuntimeClass is cluster-dependent and comes from the spec (default
+    // None). Managed clusters (GKE/EKS/AKS) expose GPUs via the device
+    // plugin with the default runtime and define no "nvidia" RuntimeClass;
+    // setting a non-existent one makes the API server reject the pod. K3s
+    // GPU nodes set it to "nvidia".
     let template = PodTemplateSpec {
         metadata: Some(ObjectMeta {
             labels: Some(labels.clone()),
             ..Default::default()
         }),
         spec: Some(PodSpec {
             containers: vec![container],
-            runtime_class_name,
+            runtime_class_name: svc.spec.runtime_class_name.clone(),
+            volumes,
             ..Default::default()
         }),
     };
