[CI] Modernize and harden Agent Stack Kubernetes installer

infra-k8s/agent-stack-values.yaml

@@ -0,0 +1,15 @@
+annotations:
+  prometheus.io/path: /metrics
+  prometheus.io/port: "9216"
+  prometheus.io/scrape: "true"
+
+config:
+  enable-completion-watcher: true
+  pod-pending-timeout: 15m
+  prometheus-port: 9216
+  reservation-expiry-seconds: 900
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 100Mi

infra-k8s/install-buildkite-k8s.sh

@@ -1,27 +1,37 @@
-set -ex
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+agentStackVersion=${AGENT_STACK_K8S_VERSION:-0.46.3}
+scriptDir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
 
 # gcloud container clusters get-credentials vllm-ci-test-cluster --region us-central1 --project vllm-405802
 
 # ensure current K8s cluster is vllm-ci-test-cluster
-kubectl config current-context | grep vllm-ci-test-cluster || (echo "Current K8s cluster is not vllm-ci-test-cluster" && exit 1)
-
+if ! kubectl config current-context | grep -q vllm-ci-test-cluster; then
+    echo "Current K8s cluster is not vllm-ci-test-cluster"
+    exit 1
+fi
 
 agentToken=${TF_VAR_buildkite_agent_token:-}
 if [ -z "$agentToken" ]; then
     echo "TF_VAR_buildkite_agent_token is not set"
     exit 1
 fi
 
-graphqlToken=${BUILDKITE_GRAPHQL_TOKEN:-}
-if [ -z "$graphqlToken" ]; then
-    echo "BUILDKITE_GRAPHQL_TOKEN is not set"
+agentQueue=${BUILDKITE_AGENT_QUEUE:-}
+if [ -z "$agentQueue" ]; then
+    echo "BUILDKITE_AGENT_QUEUE is not set"
     exit 1
 fi
 
-
 helm upgrade --install agent-stack-k8s oci://ghcr.io/buildkite/helm/agent-stack-k8s \
+    --version "$agentStackVersion" \
     --create-namespace \
     --namespace buildkite \
-    --set config.org=vllm \
-    --set agentToken=$agentToken \
-    --set graphqlToken=$graphqlToken
+    --values "$scriptDir/agent-stack-values.yaml" \
+    --set-string agentToken="$agentToken" \
+    --set-string config.queue="$agentQueue" \
+    --atomic \
+    --wait \
+    --timeout 10m