forked from llm-d/llm-d-batch-gateway
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathconfig.yaml
More file actions
90 lines (75 loc) · 3.06 KB
/
Copy pathconfig.yaml
File metadata and controls
90 lines (75 loc) · 3.06 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# Default configuration for batch-gateway API server
# This file is used for local development and testing
# Server host (empty string means all interfaces)
host: ""
# Server port
port: "8000"
# SSL certificate file path (optional)
# Uncomment and set paths to enable HTTPS
# ssl_cert_file: "path/to/cert.pem"
# ssl_key_file: "path/to/key.pem"
# Input headers for special processing (key: logical name, value: HTTP header name)
input_headers:
tenant: "X-MaaS-Username"
# HTTP server timeout configurations (in seconds)
# These values should accommodate large file uploads and batch operations
# Set to 0 or omit to use defaults
# Time allowed to read request headers (prevents Slowloris attacks)
# Default: 10 seconds
read_header_timeout_seconds: 10
# Time allowed to read the entire request (headers + body)
# Must be large enough for 200MB file uploads on slow networks
# Supports upload speeds as low as ~2.8 Mbps
# Default: 900 seconds (15 minutes)
read_timeout_seconds: 900
# Time allowed to write the response
# Must accommodate large batch query results
# Default: 120 seconds (2 minutes)
write_timeout_seconds: 120
# Time to wait for the next request on keep-alive connections
# Default: 90 seconds
idle_timeout_seconds: 90
# Max time for graceful shutdown of the API and observability HTTP servers (http.Server.Shutdown)
# Defaults: 60 seconds each; set to 0 or omit to use defaults
api_shutdown_timeout_seconds: 60
observability_shutdown_timeout_seconds: 60
# File client configuration
file_client:
# Implementation type: "fs" (default), "mock", "s3" (AWS S3)
type: "fs"
fs:
base_path: "/tmp/batch-gateway"
# S3 client configuration (required when type is "s3")
# s3:
# region: "us-west-2"
# endpoint: "" # Optional: custom endpoint for S3-compatible storage
# access_key_id: "" # Optional: defaults to AWS environment/IAM credentials
# prefix: "batch-gateway" # Optional: prefix for S3 object keys
# use_path_style: false # Optional: use path-style addressing
# Database client configuration
# db_client:
# # Type selects the database backend: "mock", "redis", or "postgresql"
# type: "redis"
#
# # Redis configuration (used when type is "redis")
# # In Kubernetes, url_secret_key references a key within the mounted secret (/etc/.secrets/).
# redis:
# url_secret_key: "redis-url" # key name within /etc/.secrets/ (mounted from existingSecret)
# Batch API configuration
batch_api:
# Batch event TTL in seconds (default: 30 days)
batch_event_ttl_seconds: 2592000
# HTTP headers to capture from batch creation requests and forward to inference.
# pass_through_headers:
# - "X-MaaS-Username"
# - "X-MaaS-Group"
# File API configuration
file_api:
# Default expiration in seconds (default: 90 days)
default_expiration_seconds: 7776000
# Maximum file size in bytes (default: 200 MB)
# Set to 0 or omit to use the default of 200 MB (209715200 bytes)
max_size_bytes: 209715200
# Maximum number of lines per file (default: 50,000)
# Set to 0 or omit to use the default of 50,000 lines
max_line_count: 50000