feat(ray-on-eks): batch LLM inference blueprint with Iceberg + tokenomics

data-stacks/ray-on-eks/examples/ray-batch-inference/01-model-staging-job.yaml

@@ -0,0 +1,97 @@
+# =============================================================================
+# Model Staging Job
+# =============================================================================
+# Downloads DeepSeek-R1-Distill-Llama-8B safetensors from HuggingFace and
+# syncs them to S3. vLLM later streams these weights directly from S3 to GPU
+# memory using the Run:ai Model Streamer (no HuggingFace download on the GPU
+# node, no giant container image layers, no EBS snapshot baking).
+#
+# Placeholders (substituted by deploy.sh):
+#   $S3_BUCKET   - S3 bucket for model weights
+#   $AWS_REGION  - AWS region
+# =============================================================================
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: model-staging-deepseek-r1-8b
+  namespace: raydata
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 3600
+  template:
+    metadata:
+      labels:
+        app: model-staging
+    spec:
+      restartPolicy: Never
+      serviceAccountName: raydata
+      containers:
+        - name: stage-model
+          image: python:3.12-slim
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              MODEL_ID="deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              S3_DEST="s3://$S3_BUCKET/models/deepseek-r1-distill-llama-8b"
+
+              echo "=== Installing tooling ==="
+              # awscli (not s5cmd): its SDK supports EKS Pod Identity credentials
+              pip install --quiet huggingface_hub awscli
+
+              # Xet-backed high-performance transfer (replaces hf_transfer)
+              export HF_XET_HIGH_PERFORMANCE=1
+              echo "=== Downloading $MODEL_ID from HuggingFace ==="
+              T0=$(date +%s)
+              hf download "$MODEL_ID" \
+                --local-dir /scratch/model \
+                --include "*.safetensors" --include "*.json" --include "*.txt"
+              T1=$(date +%s)
+              echo "TIMING hf_download_seconds=$((T1 - T0))"
+              rm -rf /scratch/model/.cache
+              du -sh /scratch/model
+
+              # Workaround for https://github.com/huggingface/transformers/issues/45488:
+              # transformers v5 LlamaTokenizer(Fast) force-installs a Metaspace
+              # pre-tokenizer that silently breaks DeepSeek R1-family tokenizers
+              # (spaces vanish on decode). Loading via PreTrainedTokenizerFast
+              # uses tokenizer.json as-is and behaves correctly.
+              python - <<'PYEOF'
+              import json
+              path = "/scratch/model/tokenizer_config.json"
+              cfg = json.load(open(path))
+              if cfg.get("tokenizer_class") == "LlamaTokenizerFast":
+                  cfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+                  json.dump(cfg, open(path, "w"), indent=1)
+                  print("Patched tokenizer_class -> PreTrainedTokenizerFast")
+              PYEOF
+
+              echo "=== Syncing to $S3_DEST ==="
+              T2=$(date +%s)
+              aws s3 sync /scratch/model/ "$S3_DEST/" --no-progress
+              T3=$(date +%s)
+              echo "TIMING s3_upload_seconds=$((T3 - T2))"
+
+              echo "=== Verifying S3 contents ==="
+              aws s3 ls "$S3_DEST/" --recursive --human-readable
+              echo "Model staging complete."
+          env:
+            - name: AWS_REGION
+              value: "$AWS_REGION"
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+              ephemeral-storage: 40Gi
+            limits:
+              memory: 12Gi
+              ephemeral-storage: 60Gi
+          volumeMounts:
+            - name: scratch
+              mountPath: /scratch
+      volumes:
+        - name: scratch
+          emptyDir:
+            sizeLimit: 50Gi
+      nodeSelector:
+        kubernetes.io/arch: amd64

data-stacks/ray-on-eks/examples/ray-batch-inference/02-image-mirror-job.yaml

@@ -0,0 +1,77 @@
+# =============================================================================
+# Image Mirror Job
+# =============================================================================
+# Mirrors the official rayproject/ray-llm image (~11.6GB) from Docker Hub to
+# a same-region private ECR repository using skopeo. Pulling from in-region
+# ECR is dramatically faster and avoids Docker Hub rate limits when GPU nodes
+# scale out.
+#
+# Prereqs (run locally, see deploy.sh):
+#   aws ecr create-repository --repository-name ray-llm --region $AWS_REGION
+#   kubectl create secret generic ecr-push-token -n raydata \
+#     --from-literal=token="$(aws ecr get-login-password --region $AWS_REGION)"
+#
+# Placeholders (substituted by deploy.sh):
+#   $ECR_REGISTRY   - e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com
+#   $RAY_LLM_TAG    - e.g. 2.56.0-py312-cu130
+# =============================================================================
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mirror-ray-llm-image
+  namespace: raydata
+spec:
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 3600
+  template:
+    metadata:
+      labels:
+        app: image-mirror
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: skopeo
+          image: quay.io/skopeo/stable:latest
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              SRC="docker://docker.io/rayproject/ray-llm:$RAY_LLM_TAG"
+              DST="docker://$ECR_REGISTRY/ray-llm:$RAY_LLM_TAG"
+              echo "=== Mirroring $SRC -> $DST ==="
+              T0=$(date +%s)
+              skopeo copy --retry-times 3 \
+                --dest-creds "AWS:$ECR_TOKEN" \
+                "$SRC" "$DST"
+              T1=$(date +%s)
+              echo "TIMING image_mirror_seconds=$((T1 - T0))"
+              echo "Image mirror complete."
+          env:
+            - name: RAY_LLM_TAG
+              value: "$RAY_LLM_TAG"
+            - name: ECR_REGISTRY
+              value: "$ECR_REGISTRY"
+            - name: ECR_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: ecr-push-token
+                  key: token
+            - name: TMPDIR
+              value: /scratch
+          resources:
+            requests:
+              cpu: "2"
+              memory: 4Gi
+              ephemeral-storage: 40Gi
+            limits:
+              memory: 8Gi
+              ephemeral-storage: 60Gi
+          volumeMounts:
+            - name: scratch
+              mountPath: /scratch
+      volumes:
+        - name: scratch
+          emptyDir:
+            sizeLimit: 50Gi
+      nodeSelector:
+        kubernetes.io/arch: amd64

data-stacks/ray-on-eks/examples/ray-batch-inference/03-rayservice-deepseek-r1-8b.yaml

@@ -0,0 +1,139 @@
+# =============================================================================
+# RayService: DeepSeek-R1-Distill-Llama-8B on vLLM 0.22 / Ray 2.56
+# =============================================================================
+# Serves an OpenAI-compatible endpoint using ray.serve.llm (build_openai_app).
+#
+# Optimizations:
+#  - Model weights stream directly from S3 to GPU memory via Run:ai Model
+#    Streamer (load_format: runai_streamer) - no HuggingFace download, no
+#    local disk staging on the GPU node.
+#  - Image pulled from same-region private ECR (mirrored from
+#    rayproject/ray-llm) - avoids Docker Hub rate limits and cross-internet
+#    transfer on every GPU node launch.
+#  - Head node is CPU-only (num-cpus: 0 so no tasks schedule on it); the
+#    vLLM engine replicas run on a dedicated g6e.2xlarge (1x L40S 48GB)
+#    worker group provisioned on demand by Karpenter.
+#
+# Placeholders (substituted by deploy.sh):
+#   $S3_BUCKET      - bucket holding model weights under models/
+#   $AWS_REGION     - AWS region
+#   $ECR_REGISTRY   - e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com
+#   $RAY_LLM_TAG    - e.g. 2.56.0-py312-cu130
+# =============================================================================
+apiVersion: ray.io/v1
+kind: RayService
+metadata:
+  name: deepseek-r1-8b
+  namespace: raydata
+spec:
+  serveConfigV2: |
+    applications:
+      - name: llm-app
+        import_path: ray.serve.llm:build_openai_app
+        route_prefix: "/"
+        args:
+          llm_configs:
+            - model_loading_config:
+                model_id: deepseek-r1-distill-llama-8b
+                model_source: s3://$S3_BUCKET/models/deepseek-r1-distill-llama-8b
+              accelerator_type: L40S
+              engine_kwargs:
+                # Stream safetensors from S3 straight to GPU memory.
+                load_format: runai_streamer
+                model_loader_extra_config:
+                  concurrency: 32
+                dtype: bfloat16
+                max_model_len: 8192
+                gpu_memory_utilization: 0.92
+                enable_prefix_caching: true
+              runtime_env:
+                # ray-llm image ships vLLM 0.22 but not the streamer extra.
+                pip:
+                  - runai-model-streamer[s3]>=0.15.7
+                env_vars:
+                  AWS_REGION: "$AWS_REGION"
+                  AWS_DEFAULT_REGION: "$AWS_REGION"
+              deployment_config:
+                autoscaling_config:
+                  min_replicas: 1
+                  max_replicas: 2
+                  target_ongoing_requests: 32
+                max_ongoing_requests: 128
+  rayClusterConfig:
+    rayVersion: "2.56.0"
+    enableInTreeAutoscaling: true
+    autoscalerOptions:
+      version: v2
+      idleTimeoutSeconds: 300
+    headGroupSpec:
+      rayStartParams:
+        # Keep the head as pure control plane: no tasks/actors schedule here.
+        num-cpus: "0"
+      template:
+        metadata:
+          labels:
+            app: deepseek-r1-8b
+        spec:
+          serviceAccountName: raydata
+          containers:
+            - name: ray-head
+              image: $ECR_REGISTRY/ray-llm:$RAY_LLM_TAG
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: AWS_REGION
+                  value: "$AWS_REGION"
+              resources:
+                requests:
+                  cpu: "2"
+                  memory: 8Gi
+                limits:
+                  memory: 12Gi
+              ports:
+                - containerPort: 8265
+                  name: dashboard
+                - containerPort: 8000
+                  name: serve
+          nodeSelector:
+            kubernetes.io/arch: amd64
+    workerGroupSpecs:
+      - groupName: gpu-workers
+        replicas: 1
+        minReplicas: 1
+        maxReplicas: 2
+        rayStartParams: {}
+        template:
+          metadata:
+            labels:
+              app: deepseek-r1-8b
+          spec:
+            serviceAccountName: raydata
+            containers:
+              - name: ray-worker
+                image: $ECR_REGISTRY/ray-llm:$RAY_LLM_TAG
+                imagePullPolicy: IfNotPresent
+                env:
+                  - name: AWS_REGION
+                    value: "$AWS_REGION"
+                resources:
+                  requests:
+                    cpu: "6"
+                    memory: 48Gi
+                    nvidia.com/gpu: "1"
+                  limits:
+                    memory: 54Gi
+                    nvidia.com/gpu: "1"
+            nodeSelector:
+              karpenter.sh/nodepool: gpu
+            # Prefer g6e.2xlarge but allow g6e.4xlarge as fallback (same
+            # single L40S 48GB GPU) - g6e.2xlarge frequently has capacity
+            # shortages (ICE) in us-west-2.
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                    - matchExpressions:
+                        - key: node.kubernetes.io/instance-type
+                          operator: In
+                          values:
+                            - g6e.2xlarge
+                            - g6e.4xlarge