cpu.yaml

# CPU Example Scenario for LLM-D Deployment
# Migrated from bash environment variables to YAML format
# Based on scenarios/examples/cpu.sh
#
# This scenario deploys a model on CPU-only nodes without GPU accelerators.
# Uses the vLLM CPU release image with large shared memory and CPU allocation.
#
# IMPORTANT NOTE:
# All parameters not defined here will use the default values from defaults.yaml.
# Many commonly defined values were left blank (default) so that this scenario
# is applicable to as many environments as possible.
#
# The model parameter is left commented -- uncomment the model you want to deploy.
# Uncommenting a model here will override the default model from defaults.yaml.
scenario:
  # ============================================================================
  # CPU Example - CPU-only deployment with no GPU accelerators
  # ============================================================================
  - name: "cpu-example"

    # =========================================================================
    # COMMON -- Applies to all deployment methods
    # =========================================================================

    # -------------------------------------------------------------------------
    # Model Configuration
    # Corresponds to: LLMDBENCH_DEPLOY_MODEL_LIST (commented -- user selects)
    # -------------------------------------------------------------------------
    # -------------------------------------------------------------------------
    # Model Configuration
    # Alternative models (uncomment and update shortName/path/huggingfaceId):
    #   Qwen/Qwen3-0.6B, Qwen/Qwen3-32B, meta-llama/Llama-3.1-8B-Instruct,
    #   meta-llama/Llama-3.1-70B-Instruct, deepseek-ai/DeepSeek-R1-0528
    # -------------------------------------------------------------------------
    model:
      name: facebook/opt-125m
      shortName: facebook-opt-125m
      path: models/facebook/opt-125m
      huggingfaceId: facebook/opt-125m
      gpuMemoryUtilization: 0
      maxModelLen: 1024

    # -------------------------------------------------------------------------
    # PVC Configuration
    # Corresponds to: LLMDBENCH_VLLM_COMMON_PVC_STORAGE_CLASS (left for auto-detect)
    # -------------------------------------------------------------------------
    # Uncomment to override storage class:
    # storage:
    #   workloadPvc:
    #     storageClassName: standard-rwx       # GKE
    #   modelPvc:
    #     storageClassName: standard-rwx       # GKE
    #     # storageClassName: shared-vast       # Kubernetes
    #     # storageClassName: ocs-storagecluster-cephfs  # OpenShift
    #     size: 1Ti
    # -------------------------------------------------------------------------
    # Affinity / Node Selection
    # Corresponds to: LLMDBENCH_VLLM_COMMON_AFFINITY=kubernetes.io/os:linux
    # By default, affinity is disabled and pods schedule on any available node.
    # Options:
    #   1. Leave commented out -- no node selection constraint
    #   2. Set nodeSelector to "auto" -- auto-detects GPU labels from the cluster
    #   3. Set explicit labels -- pods only schedule on matching nodes
    # This scenario targets CPU-only nodes (no GPU required).
    # -------------------------------------------------------------------------
    affinity:
      enabled: true
      nodeSelector:
        kubernetes.io/os: linux

    # -------------------------------------------------------------------------
    # Accelerator Configuration -- no GPUs
    # Corresponds to: LLMDBENCH_VLLM_COMMON_ACCELERATOR_NR=0
    # -------------------------------------------------------------------------
    accelerator:
      count: 0

    # -------------------------------------------------------------------------
    # vLLM Common Configuration
    # Shared settings for both decode and prefill pods.
    # When no customCommand is set on decode/prefill, the vLLM serve command
    # is auto-generated from these fields:
    #   - model.maxModelLen, model.blockSize maps to --max-model-len, --block-size
    #   - model.gpuMemoryUtilization maps to --gpu-memory-utilization
    #   - decode/prefill.parallelism.tensor maps to --tensor-parallel-size
    #   - vllmCommon.flags.* maps to --enforce-eager, --no-enable-log-requests, etc.
    #   - vllmCommon.kvTransfer.* maps to --kv-transfer-config (if enabled)
    #   - vllmCommon.kvEvents.* maps to --kv-events-config (if enabled)
    #   - decode/prefill.vllm.additionalFlags maps to appended as extra CLI args
    # When decode.vllm.customCommand or prefill.vllm.customCommand IS set, it replaces the auto-generated
    # command entirely. Only --kv-transfer-config and --kv-events-config are
    # still appended from vllmCommon if enabled. All other vllmCommon.flags
    # are ignored -- the custom command must include its own flags.
    # -------------------------------------------------------------------------
    vllmCommon:
      # Shell used in the vLLM container `command:` field. Defaults to /bin/bash.
      # shell: /bin/sh

      shmMemory: 16Gi

      preprocessScript: "python3 /setup/preprocess/set_llmdbench_environment.py && . $HOME/llmdbench_env.sh"

      # Shared volumes for CPU scenarios
      volumes:
        - name: dshm
          type: emptyDir
          emptyDir:
            medium: Memory
            sizeLimit: 16Gi
        - name: preprocesses
          type: configMap
          configMap:
            name: llm-d-benchmark-preprocesses
            defaultMode: 320
        - name: k8s-llmdbench-context
          type: secret
          secret:
            secretName: llmdbench-context

      volumeMounts:
        - name: dshm
          mountPath: /dev/shm
        - name: preprocesses
          mountPath: /setup/preprocess
        - name: k8s-llmdbench-context
          mountPath: /etc/kubeconfig
          readOnly: true

      # These flags control the auto-generated vllm serve command.
      # Flags not listed here inherit from defaults.yaml (e.g. allowLongMaxModelLen, serverDevMode).
      # To use a fully custom command instead, set decode.vllm.customCommand.
      flags:
        enforceEager: false          # old cpu.sh did not use --enforce-eager
        disableLogRequests: true     # modelservice only -- upstream vllm-openai (v0.9+) dropped --no-enable-log-requests
        disableUvicornAccessLog: true  # --disable-uvicorn-access-log
        noPrefixCaching: true        # --no-enable-prefix-caching

    # =========================================================================
    # DEPLOYMENT METHOD -- Choose standalone or modelservice
    # =========================================================================
    modelservice:
      enabled: true

    # =========================================================================
    # GATEWAY -- toggle the router topology
    # -------------------------------------------------------------------------
    # `epponly` is the llm-d "Standalone Mode" router topology
    # (see https://github.com/llm-d/llm-d/blob/main/guides/recipes/router/README.md):
    # no Kubernetes Gateway is installed, the router Helm chart is swapped
    # from `llm-d-router-gateway-dev` to `llm-d-router-standalone-dev`,
    # and the EPP pod runs an Envoy sidecar that serves HTTP directly on
    # the `{model_id_label}-router-epp`
    # Service (port 80 -> 8081). HTTPRoute and `llm-d-infra` are not
    # rendered, no gateway-provider control plane is installed.
    #
    # Flip this to any of the gateway-backed options to switch topology
    # without changing anything else in the scenario:
    #   epponly                     -- no Gateway (default here)
    #   istio                       -- istio-base + istiod + Gateway + HTTPRoute
    #   agentgateway                -- agentgateway controller + Gateway + HTTPRoute
    #   gke                         -- GKE-managed Gateway + HTTPRoute
    #   data-science-gateway-class  -- OpenDataHub managed Gateway + HTTPRoute
    #
    # This setting is ignored when `kustomize.enabled: true` (or when
    # `-t kustomize` is passed) -- in that mode the upstream llm-d guide
    # manifests are applied verbatim and define their own router topology.
    # =========================================================================
    gateway:
      className: epponly

    # =========================================================================
    # STANDALONE -- Only applies when standalone.enabled: true
    # =========================================================================

    # -------------------------------------------------------------------------
    # Standalone Deployment Configuration
    # Corresponds to: LLMDBENCH_VLLM_STANDALONE_*
    # -------------------------------------------------------------------------
    standalone:
      enabled: false
      replicas: 1

      acceleratorType:
        labelKey: kubernetes.io/os
        labelValue: linux

      image:
        repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo
        tag: v0.12.0
        pullPolicy: IfNotPresent

      accelerator:
        count: 0

      acceleratorType:
        labelKey: kubernetes.io/os
        labelValue: linux

      resources:
        limits:
          memory: 8Gi
          cpu: "2"
        requests:
          memory: 8Gi
          cpu: "2"


    # =========================================================================
    # COMMON -- Applies to all deployment methods (continued)
    # =========================================================================

    # -------------------------------------------------------------------------
    # Override images for modelservice (CPU vLLM image)
    # Corresponds to: LLMDBENCH_LLMD_IMAGE_*
    # -------------------------------------------------------------------------
    images:
      vllm:
        repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo
        tag: v0.12.0

    # =========================================================================
    # MODELSERVICE -- Only applies when modelservice.enabled: true
    # The following sections configure the llm-d modelservice Helm chart
    # deployment including decode pods, prefill pods, routing, and GAIE.
    # =========================================================================

    # -------------------------------------------------------------------------
    # Prefill Configuration -- disabled (0 prefill pods)
    # Corresponds to: LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS=0
    # -------------------------------------------------------------------------
    prefill:
      enabled: false
      replicas: 0
      accelerator:
        count: 0

    # -------------------------------------------------------------------------
    # Decode Configuration -- 2 decode pods on CPU
    # Corresponds to: LLMDBENCH_VLLM_MODELSERVICE_DECODE_*
    # No customCommand -- the vLLM serve command is auto-generated
    # from model.*, vllmCommon.flags.*, and decode.vllm.additionalFlags.
    # -------------------------------------------------------------------------
    decode:
      replicas: 2

      parallelism:
        tensor: 1
        data: 1
        dataLocal: 1
        workers: 1

      resources:
        limits:
          memory: 8Gi
          cpu: "2"
        requests:
          memory: 8Gi
          cpu: "2"

      acceleratorType:
        labelKey: kubernetes.io/os
        labelValue: linux

      accelerator:
        count: 0


    # =========================================================================
    # WORKLOAD / HARNESS -- Benchmark execution configuration
    # =========================================================================

    # -------------------------------------------------------------------------
    # Workload/Harness Configuration
    # Corresponds to: LLMDBENCH_HARNESS_NAME=inference-perf
    # -------------------------------------------------------------------------
    # Corresponds to: LLMDBENCH_CONTROL_WORK_DIR=~/data/cpu
    # Used as workspace when --workspace is not specified on the CLI.
    workDir: "~/data/cpu"

    harness:
      name: inference-perf
      experimentProfile: sanity_random.yaml