forked from llm-d/llm-d-benchmark
-
Notifications
You must be signed in to change notification settings - Fork 0
196 lines (184 loc) · 5.75 KB
/
Copy pathci-pr-benchmark.yaml
File metadata and controls
196 lines (184 loc) · 5.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
name: CI - PR Benchmark Run
on:
pull_request:
jobs:
# -- Unit tests (no cluster needed) --
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.0
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install llmdbenchmark
run: |
./install.sh -y 2>&1
pip install pytest
- name: Run unit tests
run: python -m pytest tests/ -x -q
# -- Standalone deployment (own Kind cluster) --
benchmark-standalone:
name: Benchmark - Standalone (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: cicd
standup_method: standalone
standup_scenario: kind
decode_pods: auto
prefill_pods: auto
accelerator_type: X
backend_type: X
infra_provider: X
connector: X
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: none
monitoring_enabled: true
dry_run: false
verbose: false
harness: vllm-benchmark
workload: sanity_random.yaml
# harness: nop
# workload: nop.yaml
secrets: inherit
# -- FMA deployment (own Kind cluster) --
benchmark-fma:
name: Benchmark - FMA (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: cicd
standup_method: fma
standup_scenario: kind
decode_pods: auto
prefill_pods: auto
accelerator_type: X
backend_type: X
infra_provider: X
connector: X
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: none
monitoring_enabled: true
dry_run: false
verbose: false
harness: nop
workload: nop.yaml
secrets: inherit
benchmark-modelservice-istio:
name: Benchmark - Modelservice / istio (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: cicd
standup_method: modelservice
standup_scenario: kind
decode_pods: auto
prefill_pods: auto
accelerator_type: X
backend_type: X
infra_provider: X
connector: X
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: istio
monitoring_enabled: true
dry_run: false
verbose: false
harness: inference-perf
workload: sanity_random.yaml
secrets: inherit
benchmark-modelservice-agentgateway:
name: Benchmark - Modelservice / agentgateway (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: cicd
standup_method: modelservice
standup_scenario: kind
decode_pods: auto
prefill_pods: auto
accelerator_type: X
backend_type: X
infra_provider: X
connector: X
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: agentgateway
monitoring_enabled: true
dry_run: false
verbose: false
harness: guidellm
workload: sanity_random.yaml
secrets: inherit
benchmark-modelservice-epponly:
name: Benchmark - Modelservice / epponly (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: cicd
standup_method: modelservice
standup_scenario: kind
decode_pods: auto
prefill_pods: auto
accelerator_type: X
backend_type: X
infra_provider: X
connector: X
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: epponly
monitoring_enabled: true
dry_run: false
verbose: false
harness: guidellm
workload: sanity_random.yaml
secrets: inherit
benchmark-kustomize-optimized-baseline:
name: Benchmark - kustomize / epponly / optimized-baseline (Kind)
uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
with:
scenario_dir: guides
standup_method: kustomize
standup_scenario: optimized-baseline
decode_pods: 2
prefill_pods: auto
accelerator_type: gpu
backend_type: vllm
infra_provider: base
connector:
cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }}
helm_release: lmdbenchcicdr
workspace_dir: tmp/cicd-sa-${{ github.event.number }}
bucket_project: llm-d-scale
bucket_provider: none
bucket_path: /tmp/llmdbenchcicd
gateway_class: epponly
monitoring_enabled: true
dry_run: true
verbose: false
harness: inference-perf
workload: sanity_random.yaml
secrets: inherit