forked from llm-d/llm-d-batch-gateway
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathconfig.yaml
More file actions
90 lines (82 loc) · 3.4 KB
/
Copy pathconfig.yaml
File metadata and controls
90 lines (82 loc) · 3.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# Worker Settings - task wait time needs to be shorter than poll interval
task_wait_time: "1s"
poll_interval: "5s"
num_workers: 20
queue_time_bucket:
start: 0.1
factor: 2
count: 10
process_time_bucket:
start: 0.1
factor: 2
count: 15
e2e_latency_bucket:
start: 1
factor: 3
count: 12
# Concurrency Control
concurrency:
global: 100 # Fixed ceiling for total in-flight requests
per_endpoint: 10 # Initial/max concurrency per inference endpoint
recovery: 5 # Max concurrent job recoveries during startup
aimd:
enabled: true # Set to false to disable adaptive concurrency
min: 5 # AIMD floor per endpoint
backoff_factor: 0.5 # Multiplicative decrease on 429/5xx (must be in (0,1))
additive_increase: 1 # Slots added per successful window
# Metrics & Health Check Server Address - ":PORT" format (e.g. ":9090")
addr: ":9090"
# Observability server failure should not terminate the processor in best-effort mode by default.
terminate_on_observability_failure: false
# Shutdown timeout for processor (should be less than k8s terminationGracePeriodSeconds)
shutdown_timeout: "30s"
# Work directory for processor
work_dir: "/var/lib/batch-gateway/processor"
# Inference gateway configuration.
# Either global_inference_gateway or model_gateways must be configured.
#
# Option 1: global_inference_gateway routes ALL requests to one endpoint.
# Use for MaaS / multi-model platforms or LoRA-only deployments
# where all adapters share a single vLLM instance.
# When set, model_gateways entries are ignored for routing.
global_inference_gateway:
url: "http://localhost:8000"
# inference_objective: "" # InferenceObjective CRD name for x-gateway-inference-objective header
request_timeout: "5m"
max_retries: 3
initial_backoff: "1s"
max_backoff: "60s"
tls_insecure_skip_verify: false
# For HTTPS with a private CA or mTLS, set paths to PEM files on disk (e.g. mounted in Kubernetes):
# tls_ca_cert_file: "/etc/inference-tls/ca.crt"
# tls_client_cert_file: "/etc/inference-tls/client.crt"
# tls_client_key_file: "/etc/inference-tls/client.key"
#
# Option 2: model_gateways maps specific model names to gateway configs.
# Only models listed here are routed; unlisted models get a request-level error.
# api_key_name and api_key_file are mutually exclusive.
# model_gateways:
# "llama-3":
# url: "http://gateway-a:8000"
# inference_objective: "" # InferenceObjective CRD name for x-gateway-inference-objective header
# request_timeout: "5m"
# max_retries: 3
# initial_backoff: "1s"
# max_backoff: "60s"
# tls_insecure_skip_verify: false
# # tls_ca_cert_file: "/path/to/ca-cert.pem"
# # tls_client_cert_file: "/path/to/client-cert.pem"
# # tls_client_key_file: "/path/to/client-key.pem"
# "mistral":
# url: "http://gateway-b:8000"
# api_key_name: "gateway-b-api-key"
# request_timeout: "2m"
# max_retries: 1
# Default TTL for output/error files in seconds (0 = no expiration).
# Used as fallback when the user does not provide output_expires_after in POST /v1/batches.
default_output_expiration_seconds: 7776000 # 90 days
# TTL for temporary progress updates in the status store (Redis), in seconds.
progress_ttl_seconds: 86400 # 24 hours
# Whether to send x-gateway-inference-fairness-id on inference requests.
# false (default) omits the fairness header entirely.
send_fairness_header: false