Skip to content

Commit a95def0

Browse files
vara-bonthuvara-bonthu
authored
feat(ray-on-eks): batch LLM inference blueprint with Iceberg + tokenomics (#1124)
1 parent ec3e871 commit a95def0

24 files changed

Lines changed: 2789 additions & 15 deletions

‎data-stacks/ray-on-eks/examples/ray-batch-inference/01-model-staging-job.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# =============================================================================
2+
# Model Staging Job
3+
# =============================================================================
4+
# Downloads DeepSeek-R1-Distill-Llama-8B safetensors from HuggingFace and
5+
# syncs them to S3. vLLM later streams these weights directly from S3 to GPU
6+
# memory using the Run:ai Model Streamer (no HuggingFace download on the GPU
7+
# node, no giant container image layers, no EBS snapshot baking).
8+
#
9+
# Placeholders (substituted by deploy.sh):
10+
# $S3_BUCKET - S3 bucket for model weights
11+
# $AWS_REGION - AWS region
12+
# =============================================================================
13+
apiVersion: batch/v1
14+
kind: Job
15+
metadata:
16+
name: model-staging-deepseek-r1-8b
17+
namespace: raydata
18+
spec:
19+
backoffLimit: 1
20+
ttlSecondsAfterFinished: 3600
21+
template:
22+
metadata:
23+
labels:
24+
app: model-staging
25+
spec:
26+
restartPolicy: Never
27+
serviceAccountName: raydata
28+
containers:
29+
- name: stage-model
30+
image: python:3.12-slim
31+
command: ["/bin/bash", "-c"]
32+
args:
33+
- |
34+
set -euo pipefail
35+
MODEL_ID="deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
36+
S3_DEST="s3://$S3_BUCKET/models/deepseek-r1-distill-llama-8b"
37+
38+
echo "=== Installing tooling ==="
39+
# awscli (not s5cmd): its SDK supports EKS Pod Identity credentials
40+
pip install --quiet huggingface_hub awscli
41+
42+
# Xet-backed high-performance transfer (replaces hf_transfer)
43+
export HF_XET_HIGH_PERFORMANCE=1
44+
echo "=== Downloading $MODEL_ID from HuggingFace ==="
45+
T0=$(date +%s)
46+
hf download "$MODEL_ID" \
47+
--local-dir /scratch/model \
48+
--include "*.safetensors" --include "*.json" --include "*.txt"
49+
T1=$(date +%s)
50+
echo "TIMING hf_download_seconds=$((T1 - T0))"
51+
rm -rf /scratch/model/.cache
52+
du -sh /scratch/model
53+
54+
# Workaround for https://github.com/huggingface/transformers/issues/45488:
55+
# transformers v5 LlamaTokenizer(Fast) force-installs a Metaspace
56+
# pre-tokenizer that silently breaks DeepSeek R1-family tokenizers
57+
# (spaces vanish on decode). Loading via PreTrainedTokenizerFast
58+
# uses tokenizer.json as-is and behaves correctly.
59+
python - <<'PYEOF'
60+
import json
61+
path = "/scratch/model/tokenizer_config.json"
62+
cfg = json.load(open(path))
63+
if cfg.get("tokenizer_class") == "LlamaTokenizerFast":
64+
cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
65+
json.dump(cfg, open(path, "w"), indent=1)
66+
print("Patched tokenizer_class -> PreTrainedTokenizerFast")
67+
PYEOF
68+
69+
echo "=== Syncing to $S3_DEST ==="
70+
T2=$(date +%s)
71+
aws s3 sync /scratch/model/ "$S3_DEST/" --no-progress
72+
T3=$(date +%s)
73+
echo "TIMING s3_upload_seconds=$((T3 - T2))"
74+
75+
echo "=== Verifying S3 contents ==="
76+
aws s3 ls "$S3_DEST/" --recursive --human-readable
77+
echo "Model staging complete."
78+
env:
79+
- name: AWS_REGION
80+
value: "$AWS_REGION"
81+
resources:
82+
requests:
83+
cpu: "4"
84+
memory: 8Gi
85+
ephemeral-storage: 40Gi
86+
limits:
87+
memory: 12Gi
88+
ephemeral-storage: 60Gi
89+
volumeMounts:
90+
- name: scratch
91+
mountPath: /scratch
92+
volumes:
93+
- name: scratch
94+
emptyDir:
95+
sizeLimit: 50Gi
96+
nodeSelector:
97+
kubernetes.io/arch: amd64

‎data-stacks/ray-on-eks/examples/ray-batch-inference/02-image-mirror-job.yaml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# =============================================================================
2+
# Image Mirror Job
3+
# =============================================================================
4+
# Mirrors the official rayproject/ray-llm image (~11.6GB) from Docker Hub to
5+
# a same-region private ECR repository using skopeo. Pulling from in-region
6+
# ECR is dramatically faster and avoids Docker Hub rate limits when GPU nodes
7+
# scale out.
8+
#
9+
# Prereqs (run locally, see deploy.sh):
10+
# aws ecr create-repository --repository-name ray-llm --region $AWS_REGION
11+
# kubectl create secret generic ecr-push-token -n raydata \
12+
# --from-literal=token="$(aws ecr get-login-password --region $AWS_REGION)"
13+
#
14+
# Placeholders (substituted by deploy.sh):
15+
# $ECR_REGISTRY - e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com
16+
# $RAY_LLM_TAG - e.g. 2.56.0-py312-cu130
17+
# =============================================================================
18+
apiVersion: batch/v1
19+
kind: Job
20+
metadata:
21+
name: mirror-ray-llm-image
22+
namespace: raydata
23+
spec:
24+
backoffLimit: 2
25+
ttlSecondsAfterFinished: 3600
26+
template:
27+
metadata:
28+
labels:
29+
app: image-mirror
30+
spec:
31+
restartPolicy: Never
32+
containers:
33+
- name: skopeo
34+
image: quay.io/skopeo/stable:latest
35+
command: ["/bin/bash", "-c"]
36+
args:
37+
- |
38+
set -euo pipefail
39+
SRC="docker://docker.io/rayproject/ray-llm:$RAY_LLM_TAG"
40+
DST="docker://$ECR_REGISTRY/ray-llm:$RAY_LLM_TAG"
41+
echo "=== Mirroring $SRC -> $DST ==="
42+
T0=$(date +%s)
43+
skopeo copy --retry-times 3 \
44+
--dest-creds "AWS:$ECR_TOKEN" \
45+
"$SRC" "$DST"
46+
T1=$(date +%s)
47+
echo "TIMING image_mirror_seconds=$((T1 - T0))"
48+
echo "Image mirror complete."
49+
env:
50+
- name: RAY_LLM_TAG
51+
value: "$RAY_LLM_TAG"
52+
- name: ECR_REGISTRY
53+
value: "$ECR_REGISTRY"
54+
- name: ECR_TOKEN
55+
valueFrom:
56+
secretKeyRef:
57+
name: ecr-push-token
58+
key: token
59+
- name: TMPDIR
60+
value: /scratch
61+
resources:
62+
requests:
63+
cpu: "2"
64+
memory: 4Gi
65+
ephemeral-storage: 40Gi
66+
limits:
67+
memory: 8Gi
68+
ephemeral-storage: 60Gi
69+
volumeMounts:
70+
- name: scratch
71+
mountPath: /scratch
72+
volumes:
73+
- name: scratch
74+
emptyDir:
75+
sizeLimit: 50Gi
76+
nodeSelector:
77+
kubernetes.io/arch: amd64

‎data-stacks/ray-on-eks/examples/ray-batch-inference/03-rayservice-deepseek-r1-8b.yaml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# =============================================================================
2+
# RayService: DeepSeek-R1-Distill-Llama-8B on vLLM 0.22 / Ray 2.56
3+
# =============================================================================
4+
# Serves an OpenAI-compatible endpoint using ray.serve.llm (build_openai_app).
5+
#
6+
# Optimizations:
7+
# - Model weights stream directly from S3 to GPU memory via Run:ai Model
8+
# Streamer (load_format: runai_streamer) - no HuggingFace download, no
9+
# local disk staging on the GPU node.
10+
# - Image pulled from same-region private ECR (mirrored from
11+
# rayproject/ray-llm) - avoids Docker Hub rate limits and cross-internet
12+
# transfer on every GPU node launch.
13+
# - Head node is CPU-only (num-cpus: 0 so no tasks schedule on it); the
14+
# vLLM engine replicas run on a dedicated g6e.2xlarge (1x L40S 48GB)
15+
# worker group provisioned on demand by Karpenter.
16+
#
17+
# Placeholders (substituted by deploy.sh):
18+
# $S3_BUCKET - bucket holding model weights under models/
19+
# $AWS_REGION - AWS region
20+
# $ECR_REGISTRY - e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com
21+
# $RAY_LLM_TAG - e.g. 2.56.0-py312-cu130
22+
# =============================================================================
23+
apiVersion: ray.io/v1
24+
kind: RayService
25+
metadata:
26+
name: deepseek-r1-8b
27+
namespace: raydata
28+
spec:
29+
serveConfigV2: |
30+
applications:
31+
- name: llm-app
32+
import_path: ray.serve.llm:build_openai_app
33+
route_prefix: "/"
34+
args:
35+
llm_configs:
36+
- model_loading_config:
37+
model_id: deepseek-r1-distill-llama-8b
38+
model_source: s3://$S3_BUCKET/models/deepseek-r1-distill-llama-8b
39+
accelerator_type: L40S
40+
engine_kwargs:
41+
# Stream safetensors from S3 straight to GPU memory.
42+
load_format: runai_streamer
43+
model_loader_extra_config:
44+
concurrency: 32
45+
dtype: bfloat16
46+
max_model_len: 8192
47+
gpu_memory_utilization: 0.92
48+
enable_prefix_caching: true
49+
runtime_env:
50+
# ray-llm image ships vLLM 0.22 but not the streamer extra.
51+
pip:
52+
- runai-model-streamer[s3]>=0.15.7
53+
env_vars:
54+
AWS_REGION: "$AWS_REGION"
55+
AWS_DEFAULT_REGION: "$AWS_REGION"
56+
deployment_config:
57+
autoscaling_config:
58+
min_replicas: 1
59+
max_replicas: 2
60+
target_ongoing_requests: 32
61+
max_ongoing_requests: 128
62+
rayClusterConfig:
63+
rayVersion: "2.56.0"
64+
enableInTreeAutoscaling: true
65+
autoscalerOptions:
66+
version: v2
67+
idleTimeoutSeconds: 300
68+
headGroupSpec:
69+
rayStartParams:
70+
# Keep the head as pure control plane: no tasks/actors schedule here.
71+
num-cpus: "0"
72+
template:
73+
metadata:
74+
labels:
75+
app: deepseek-r1-8b
76+
spec:
77+
serviceAccountName: raydata
78+
containers:
79+
- name: ray-head
80+
image: $ECR_REGISTRY/ray-llm:$RAY_LLM_TAG
81+
imagePullPolicy: IfNotPresent
82+
env:
83+
- name: AWS_REGION
84+
value: "$AWS_REGION"
85+
resources:
86+
requests:
87+
cpu: "2"
88+
memory: 8Gi
89+
limits:
90+
memory: 12Gi
91+
ports:
92+
- containerPort: 8265
93+
name: dashboard
94+
- containerPort: 8000
95+
name: serve
96+
nodeSelector:
97+
kubernetes.io/arch: amd64
98+
workerGroupSpecs:
99+
- groupName: gpu-workers
100+
replicas: 1
101+
minReplicas: 1
102+
maxReplicas: 2
103+
rayStartParams: {}
104+
template:
105+
metadata:
106+
labels:
107+
app: deepseek-r1-8b
108+
spec:
109+
serviceAccountName: raydata
110+
containers:
111+
- name: ray-worker
112+
image: $ECR_REGISTRY/ray-llm:$RAY_LLM_TAG
113+
imagePullPolicy: IfNotPresent
114+
env:
115+
- name: AWS_REGION
116+
value: "$AWS_REGION"
117+
resources:
118+
requests:
119+
cpu: "6"
120+
memory: 48Gi
121+
nvidia.com/gpu: "1"
122+
limits:
123+
memory: 54Gi
124+
nvidia.com/gpu: "1"
125+
nodeSelector:
126+
karpenter.sh/nodepool: gpu
127+
# Prefer g6e.2xlarge but allow g6e.4xlarge as fallback (same
128+
# single L40S 48GB GPU) - g6e.2xlarge frequently has capacity
129+
# shortages (ICE) in us-west-2.
130+
affinity:
131+
nodeAffinity:
132+
requiredDuringSchedulingIgnoredDuringExecution:
133+
nodeSelectorTerms:
134+
- matchExpressions:
135+
- key: node.kubernetes.io/instance-type
136+
operator: In
137+
values:
138+
- g6e.2xlarge
139+
- g6e.4xlarge

0 commit comments

Comments
 (0)