values.yaml

extraServicePorts: []

# Kubernetes cluster DNS domain, used to build in-cluster Service FQDNs.
clusterDomain: cluster.local

epp:
  replicas: 1
  image:
    registry: ghcr.io/llm-d
    repository: llm-d-router-endpoint-picker
    tag: main
    pullPolicy: Always
  extProcPort: 9002
  # gRPC port serving EPP liveness and readiness checks, 9003 if unset.
  # grpcHealthPort: 9003

  parsers: [] # e.g. ["openai-parser", "anthropic-parser"], or empty for auto-selection

  # Metrics DataSource Configuration
  # These values configure how the EPP scrapes metrics from model server pods.
  metricsDataSource:
    # scheme is the HTTP scheme used to scrape metrics (http or https).
    scheme: "http"
    # path is the URL path on the model server pod that exposes Prometheus metrics.
    path: "/metrics"
    # insecureSkipVerify disables TLS certificate verification when scheme is https.
    insecureSkipVerify: true

  # Example flags:
  # flags:
  #  # Log verbosity
  #  v: 2
  
  # Example environment variables:
  # env:
  #   ENABLE_EXPERIMENTAL_FEATURE: "true"
  env: []
  
  pluginsConfigFile: "default-plugins.yaml"
  pluginsCustomConfig: {}
  
  # Define additional container ports
  extraContainerPorts: []
  
  # Example defining a custom EPP config file
  # pluginsCustomConfig:
  #   custom-plugins.yaml: |
  #     apiVersion: inference.networking.x-k8s.io/v1alpha1
  #     kind: EndpointPickerConfig
  #     plugins:
  #     - type: custom-scorer
  #       parameters:
  #         custom-threshold: 64
  #     schedulingProfiles:
  #     - name: default
  #       plugins:
  #       - pluginRef: custom-scorer
  
  # EPP container resources: CPU limits unset for burst capacity, memory capped at 16Gi
  resources:
    requests:
      cpu: "8"
      memory: 8Gi
    limits:
      memory: 16Gi

  affinity: {}
  tolerations: []

  # Annotations added to the EPP pod template. Useful for clusters that rely on
  # annotation-based Prometheus scraping (prometheus.io/{scrape,port,path}) or
  # Istio metrics-merging, rather than a ServiceMonitor or GMP.
  podAnnotations: {}

# By default no proxy is configured
proxy:
  enabled: false    

# Tokenizer sidecar. Runs vLLM's `vllm launch render <modelName>` and exposes
# `/render` over loopback HTTP to EPP. modelName is passed as the first
# positional arg to the render command (required when enabled). EPP is wired to
# the sidecar via `router.epp.pluginsCustomConfig`.
tokenizer:
  enabled: false
  modelName: ""
  image:
    registry: docker.io
    repository: vllm/vllm-openai-cpu
    tag: v0.19.1
    pullPolicy: IfNotPresent
  port: 8000
  command: []
  args: []
  # Extra args appended to the vllm-render container after the default or overridden args.
  extraArgs: []
  # Pod-level init containers rendered when the tokenizer is enabled.
  initContainers: []
  # Pod-level volumes rendered alongside model-cache when the tokenizer is enabled.
  volumes: []
  env:
    - name: HF_TOKEN
      valueFrom:
        secretKeyRef:
          name: llm-d-hf-token
          key: HF_TOKEN
  resources:
    requests:
      cpu: "4"
      memory: 8Gi
  volumeMounts: []
  readinessProbe:
    httpGet:
      path: /health
      port: render-http
    initialDelaySeconds: 30
    periodSeconds: 10
    timeoutSeconds: 5
    failureThreshold: 60

# Monitoring configuration for EPP
monitoring:
  interval: "10s"
  # Monitoring provider selection.
  # Options: ["gmp", "prometheusoperator"].
  #     gmp: Google Managed Prometheus
  #     prometheusoperator: Open Source Prometheus Operator
  # If not set, defaults to "prometheusoperator". For backwards compatibility,
  # legacy provider.name=gke still maps to "gmp" when this field is unset.
  provider:
    name: ""
    gmp:
      autopilot: false
  # Prometheus ServiceMonitor will be created when enabled for EPP metrics collection
  prometheus:
    enabled: false
    auth:
      # Set to false to allow unauthenticated /metrics access. Required for annotation-based
      # Prometheus scraping and Istio metrics-merging, which do not send Bearer tokens.
      enabled: true
      # Service account token secret for authentication
      secretName: inference-gateway-sa-metrics-reader-secret
    # additional labels for the ServiceMonitor
    extraLabels: {}

tracing:
  enabled: false
  otelExporterEndpoint: "http://localhost:4317"
  sampling:
    sampler: "parentbased_traceidratio"
    samplerArg: "0.1"

inferencePool:
  create: true
  failureMode: "FailOpen"

inferenceObjectives: []
# - name: high-priority
#   priority: 5
# - name: low-priority
#   priority: 1

modelServers:
  type: vllm # vllm, sglang, triton-tensorrt-llm, trtllm-serve, triton
  protocol: http # http, grpc
  targetPorts:
    - number: 8000
  # matchLabels: # REQUIRED in parent charts if creating inference pool
  #   app: vllm-qwen3-32b

latencyPredictor:
  enabled: false
  # Training Server Configuration
  trainingServer:
    image:
      registry: ghcr.io/llm-d
      repository: llm-d-latency-predictor-training-server-dev
      tag: latest
      pullPolicy: Always
    port: 8000
    resources:
      requests:
        cpu: "2000m"
        memory: "4Gi"
      limits:
        cpu: "4000m"
        memory: "8Gi"
    livenessProbe:
      httpGet:
        path: /healthz
        port: 8000
      initialDelaySeconds: 30
      periodSeconds: 20
    readinessProbe:
      httpGet:
        path: /readyz
        port: 8000
      initialDelaySeconds: 45
      periodSeconds: 10
    volumeSize: "20Gi"
    config:
      LATENCY_RETRAINING_INTERVAL_SEC: "10"
      LATENCY_MIN_SAMPLES_FOR_RETRAIN: "100"
      LATENCY_TTFT_MODEL_PATH: "/models/ttft.joblib"
      LATENCY_TPOT_MODEL_PATH: "/models/tpot.joblib"
      LATENCY_TTFT_SCALER_PATH: "/models/ttft_scaler.joblib"
      LATENCY_TPOT_SCALER_PATH: "/models/tpot_scaler.joblib"
      LATENCY_MODEL_TYPE: "xgboost"
      LATENCY_MAX_TRAINING_DATA_SIZE_PER_BUCKET: "500"
      LATENCY_OBJECTIVE_TYPE: "mean"

  # Prediction Server Configuration
  predictionServers:
    count: 1
    startPort: 8001
    image:
      registry: ghcr.io/llm-d
      repository: llm-d-latency-predictor-prediction-server-dev
      tag: latest
      pullPolicy: Always
    resources:
      requests:
        cpu: "8000m"
        memory: "4Gi"
      limits:
        cpu: "28000m"
        memory: "8Gi"
    livenessProbe:
      httpGet:
        path: /healthz
      initialDelaySeconds: 15
      periodSeconds: 15
      timeoutSeconds: 5
      failureThreshold: 5
    readinessProbe:
      httpGet:
        path: /readyz
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 5
      failureThreshold: 30
    volumeSize: "10Gi"
    config:
      LATENCY_MODEL_TYPE: "xgboost"
      PREDICT_HOST: "0.0.0.0"
      LOCAL_TTFT_MODEL_PATH: "/server_models/ttft.joblib"
      LOCAL_TPOT_MODEL_PATH: "/server_models/tpot.joblib"
      LOCAL_TTFT_SCALER_PATH: "/server_models/ttft_scaler.joblib"
      LOCAL_TPOT_SCALER_PATH: "/server_models/tpot_scaler.joblib"
      UVICORN_WORKERS: "28"
      OMP_NUM_THREADS: "1"
      MODEL_SYNC_INTERVAL_SEC: "30"
      LATENCY_OBJECTIVE_TYPE: "mean"

  # EPP Environment Variables for Latency Predictor
  eppEnv:
    LATENCY_MAX_SAMPLE_SIZE: "10000"
    LATENCY_MAX_CONCURRENT_DISPATCHES: "36"
    LATENCY_COALESCE_WINDOW_MS: "1"