forked from llm-d/llm-d-workload-variant-autoscaler
-
Notifications
You must be signed in to change notification settings - Fork 0
617 lines (554 loc) · 26.4 KB
/
Copy pathci-e2e-openshift.yaml
File metadata and controls
617 lines (554 loc) · 26.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
name: CI - OpenShift E2E Tests
# Permissions needed for various jobs
permissions:
contents: read
packages: write
pull-requests: write # For posting comments on PRs
statuses: write # For reporting status on fork PR commits
# Cancel previous runs on the same PR to avoid resource conflicts
# Only group by PR number for legitimate triggers (pull_request, workflow_dispatch, /ok-to-test, or /retest comments)
# Regular comments get a unique group (run_id) so they don't cancel in-progress test runs
#
# Logic:
# - Regular comments (not /ok-to-test or /retest): unique group prevents cancellation of real tests
# - Valid triggers: group 'e2e-openshift-{pr_number}' (can cancel previous runs for same PR)
# - Fallback chain for ID: pull_request.number -> issue.number -> run_id
#
# NOTE: Valid command list (/ok-to-test, /retest) must stay in sync with gate job validation (line ~125)
concurrency:
group: >-
${{
github.event_name == 'issue_comment' &&
!contains(github.event.comment.body, '/ok-to-test') &&
!contains(github.event.comment.body, '/retest')
&& format('comment-isolated-{0}', github.run_id)
|| format('e2e-openshift-{0}',
github.event.pull_request.number
|| github.event.issue.number
|| github.run_id)
}}
cancel-in-progress: true
on:
pull_request:
branches:
- main
- dev
# Allow maintainers to trigger tests on fork PRs via /ok-to-test comment
issue_comment:
types: [created]
workflow_dispatch:
inputs:
model_id:
description: 'Model ID'
required: false
default: 'e2ewva/dummy-model'
accelerator_type:
description: 'Accelerator type (H100, A100, L40S)'
required: false
default: 'H100'
request_rate:
description: 'Request rate (req/s)'
required: false
default: '20'
num_prompts:
description: 'Number of prompts'
required: false
default: '3000'
skip_cleanup:
description: 'Skip cleanup after tests'
required: false
default: 'false'
max_num_seqs:
description: 'vLLM max batch size (lower = easier to saturate)'
required: false
default: '1'
hpa_stabilization_seconds:
description: 'HPA stabilization window in seconds'
required: false
default: '240'
jobs:
# Check if PR contains code changes (not just docs/metadata)
check-code-changes:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
has_code_changes: ${{ steps.set-output.outputs.has_code_changes }}
steps:
- name: Checkout source
if: github.event_name == 'pull_request'
uses: actions/checkout@v6
- name: Check for code changes
if: github.event_name == 'pull_request'
uses: dorny/paths-filter@v3
id: filter
with:
predicate-quantifier: 'every'
filters: |
code:
- '**'
- '!docs/**'
- '!README.md'
- '!CONTRIBUTING.md'
- '!LICENSE'
- '!OWNERS'
- '!PROJECT'
- name: Set output
id: set-output
run: |
if [ "${{ github.event_name }}" != "pull_request" ]; then
# Always run for issue_comment (/ok-to-test, /retest) and workflow_dispatch
echo "has_code_changes=true" >> $GITHUB_OUTPUT
elif [ -n "${{ steps.filter.outputs.code }}" ]; then
echo "has_code_changes=${{ steps.filter.outputs.code }}" >> $GITHUB_OUTPUT
else
echo "has_code_changes=true" >> $GITHUB_OUTPUT
fi
# Gate: Check permissions and handle /ok-to-test for fork PRs.
# - Maintainers (write access): Tests run automatically on pull_request.
# - Fork PRs: Gate succeeds (no failure) so the PR does not show a false red check; E2E runs
# only after a maintainer comments /ok-to-test. Branch protection should require the
# "e2e-openshift" job so merge stays blocked until that run passes.
gate:
needs: check-code-changes
if: needs.check-code-changes.outputs.has_code_changes == 'true'
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
pr_number: ${{ steps.check.outputs.pr_number }}
pr_head_sha: ${{ steps.check.outputs.pr_head_sha }}
is_fork_pr: ${{ steps.check.outputs.is_fork_pr }}
steps:
- name: Check permissions and OpenShift E2E triggers (/ok-to-test, /retest)
id: check
uses: actions/github-script@v7
with:
script: |
// Helper to check if user has write access
async function hasWriteAccess(username) {
try {
const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: username
});
const privilegedRoles = ['admin', 'maintain', 'write'];
return privilegedRoles.includes(permission.permission);
} catch (e) {
console.log(`Could not get permissions for ${username}: ${e.message}`);
return false;
}
}
// Always run for workflow_dispatch
if (context.eventName === 'workflow_dispatch') {
core.setOutput('should_run', 'true');
core.setOutput('pr_number', '');
core.setOutput('pr_head_sha', context.sha);
core.setOutput('is_fork_pr', 'false');
return;
}
// Handle issue_comment event (/ok-to-test or /retest)
if (context.eventName === 'issue_comment') {
const comment = context.payload.comment.body.trim();
const issue = context.payload.issue;
// Only process /ok-to-test or /retest comments on PRs
if (!issue.pull_request) {
console.log('Comment is not on a PR, skipping');
core.setOutput('should_run', 'false');
return;
}
// NOTE: This list must stay in sync with concurrency group logic (lines 23-25)
const validCommands = ['/ok-to-test', '/retest'];
if (!validCommands.includes(comment)) {
console.log(`Comment "${comment}" is not a valid trigger command, skipping`);
core.setOutput('should_run', 'false');
return;
}
// Check if commenter has write access
const commenter = context.payload.comment.user.login;
const hasAccess = await hasWriteAccess(commenter);
if (!hasAccess) {
console.log(`User ${commenter} does not have write access, ignoring ${comment}`);
core.setOutput('should_run', 'false');
return;
}
// Get PR details to get head SHA
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: issue.number
});
// Check if PR is from a fork
const baseRepo = `${context.repo.owner}/${context.repo.repo}`;
const headRepo = pr.head.repo ? pr.head.repo.full_name : baseRepo;
const isFork = headRepo !== baseRepo;
console.log(`${comment} approved by ${commenter} for PR #${issue.number}`);
console.log(`PR head SHA: ${pr.head.sha}`);
console.log(`Is fork PR: ${isFork} (head: ${headRepo}, base: ${baseRepo})`);
core.setOutput('should_run', 'true');
core.setOutput('pr_number', issue.number.toString());
core.setOutput('pr_head_sha', pr.head.sha);
core.setOutput('is_fork_pr', isFork ? 'true' : 'false');
// Add reaction to acknowledge
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'rocket'
});
// Post comment with link to the e2e workflow run
const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const cmdDesc = comment === '/ok-to-test' ? 'approve and run' : 're-run';
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
body: `🚀 **OpenShift E2E** — ${cmdDesc} (\`${comment}\`)\n\n[View the OpenShift E2E workflow run](${runUrl})`
});
return;
}
// Handle pull_request event
const pr = context.payload.pull_request;
const prAuthor = pr.user.login;
const prNumber = pr.number;
const prHeadSha = pr.head.sha;
// Check if PR is from a fork
const baseRepo = `${context.repo.owner}/${context.repo.repo}`;
const headRepo = pr.head.repo ? pr.head.repo.full_name : baseRepo;
const isFork = headRepo !== baseRepo;
console.log(`PR #${prNumber} is from fork: ${isFork} (head: ${headRepo}, base: ${baseRepo})`);
core.setOutput('pr_number', prNumber.toString());
core.setOutput('pr_head_sha', prHeadSha);
core.setOutput('is_fork_pr', isFork ? 'true' : 'false');
// Check if PR author has write access
const isPrivileged = await hasWriteAccess(prAuthor);
console.log(`PR #${prNumber} author ${prAuthor}: privileged=${isPrivileged}`);
// Check if we already posted a bot comment
const comments = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber
});
const botComment = comments.data.find(c =>
c.user.type === 'Bot' &&
c.body.includes('ok-to-test')
);
// Helper to safely post a comment (may fail on fork PRs due to permissions)
async function tryPostComment(body) {
try {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: body
});
return true;
} catch (e) {
// Fork PRs can't post comments on pull_request event (GitHub security restriction)
console.log(`Could not post comment (expected for fork PRs): ${e.message}`);
return false;
}
}
if (isPrivileged) {
// For maintainer/admin fork PRs, we need to trigger via /ok-to-test
// because fork PRs don't have access to secrets on pull_request event
if (isFork) {
console.log(`Maintainer fork PR detected - auto-triggering /ok-to-test for ${prAuthor}`);
core.setOutput('should_run', 'false'); // Don't run on pull_request event
// Auto-post /ok-to-test to trigger issue_comment workflow
if (!botComment) {
const posted = await tryPostComment(`/ok-to-test`);
if (!posted) {
console.log('Note: Maintainer will need to manually comment /ok-to-test');
}
}
// Do not fail the gate: fork PRs cannot run E2E on pull_request (no secrets).
// Gate succeeds so the PR does not show a false failure; branch protection
// should require "e2e-openshift" so merge stays blocked until /ok-to-test run passes.
return;
}
// Non-fork PR from maintainer - run directly
core.setOutput('should_run', 'true');
return;
}
// External contributor - post instructions and skip
console.log('External contributor PR - posting instructions');
core.setOutput('should_run', 'false');
if (!botComment) {
const posted = await tryPostComment(`👋 Thanks for your contribution!\n\nThis PR is from a fork, so **OpenShift E2E** (GPU) tests require approval to run.\n\n**For maintainers/admins:** Comment \`/ok-to-test\` to approve and trigger **OpenShift E2E** on this PR, or \`/retest\` to re-run OpenShift E2E (e.g. after a failure or new commits).\n\n**For contributors:** Please wait for a maintainer or admin to approve running the tests.`);
if (!posted) {
console.log('Note: Could not post instructions comment on fork PR');
}
}
// Do not fail the gate: GitHub does not allow updating status from upstream on fork
// PRs, so a failed gate would stay red even after /ok-to-test run passes. Let the gate
// succeed; branch protection should require "e2e-openshift" so merge stays blocked
// until a maintainer comments /ok-to-test and E2E passes.
- name: Write workflow summary
if: always()
uses: actions/github-script@v7
with:
script: |
const shouldRun = '${{ steps.check.outputs.should_run }}';
const isFork = '${{ steps.check.outputs.is_fork_pr }}';
const eventName = '${{ github.event_name }}';
if (shouldRun === 'true') {
core.summary.addRaw('✅ **E2E tests will run** for this trigger.\n').write();
} else if (isFork === 'true' && eventName === 'pull_request') {
core.summary.addRaw([
'⏸️ **E2E tests skipped — fork PR**\n\n',
'Fork PRs cannot run E2E on `pull_request` events (no access to secrets/GPU runners).\n\n',
'A maintainer must comment \`/ok-to-test\` to trigger the **OpenShift E2E** suite. ',
'Branch protection should require **e2e-openshift** so merge stays blocked until E2E passes.\n',
].join('')).write();
} else {
core.summary.addRaw('⏸️ **E2E tests were skipped** (gate check did not pass for this trigger).\n').write();
}
# Build the WVA controller image on GitHub-hosted runner (has proper Docker setup)
# Note: Skip for fork PRs on pull_request event (no secrets access).
# For fork PRs, build-image runs via issue_comment trigger (/ok-to-test).
build-image:
needs: gate
if: |
needs.gate.outputs.should_run == 'true' &&
(needs.gate.outputs.is_fork_pr != 'true' || github.event_name != 'pull_request')
runs-on: ubuntu-latest
outputs:
image_tag: ${{ steps.build.outputs.image_tag }}
steps:
- name: Checkout source
uses: actions/checkout@v6
with:
# Use PR head SHA from gate (works for both pull_request and issue_comment)
ref: ${{ needs.gate.outputs.pr_head_sha }}
fetch-depth: 0
- name: Merge main into PR branch
if: github.event_name == 'issue_comment'
run: |
git config user.email "ci@github.com"
git config user.name "CI"
git remote add upstream ${{ github.server_url }}/${{ github.repository }}.git
git fetch upstream main
git merge upstream/main --no-edit
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.CR_USER }}
password: ${{ secrets.CR_TOKEN }}
- name: Build and push image
id: build
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
# Use PR head SHA from gate
GIT_REF: ${{ needs.gate.outputs.pr_head_sha }}
run: deploy/ci-e2e-openshift/build-push-image.sh
# Run e2e tests on OpenShift self-hosted runner (vllm-d cluster).
# pok-prod runners are reserved for nightly E2E only.
e2e-openshift:
runs-on: [self-hosted, openshift, vllm-d]
needs: [gate, build-image]
if: needs.gate.outputs.should_run == 'true'
env:
MODEL_ID: ${{ github.event.inputs.model_id || 'e2ewva/dummy-model' }}
GOTOOLCHAIN: auto
ACCELERATOR_TYPE: ${{ github.event.inputs.accelerator_type || 'A100' }}
REQUEST_RATE: ${{ github.event.inputs.request_rate || '20' }}
NUM_PROMPTS: ${{ github.event.inputs.num_prompts || '3000' }}
MAX_NUM_SEQS: ${{ github.event.inputs.max_num_seqs || '5' }}
HPA_STABILIZATION_SECONDS: ${{ github.event.inputs.hpa_stabilization_seconds || '240' }}
SKIP_CLEANUP: ${{ github.event.inputs.skip_cleanup || 'false' }}
LLM_D_ROUTER_VERSION: v0.9.0
GAIE_VERSION: v1.5.0
# PR-specific namespaces for isolation between concurrent PR tests
LLMD_NAMESPACE: llm-d-autoscaler-pr-${{ needs.gate.outputs.pr_number || github.run_id }}
WVA_NAMESPACE: llm-d-autoscaler-pr-${{ needs.gate.outputs.pr_number || github.run_id }}
# Unique release names per run to avoid conflicts
WVA_RELEASE_NAME: wva-e2e-${{ github.run_id }}
MODEL_A1_RELEASE: model-a1-${{ github.run_id }}
# Use the image built in the previous job
WVA_IMAGE_TAG: ${{ needs.build-image.outputs.image_tag }}
steps:
- name: Checkout source
uses: actions/checkout@v6
with:
# Use PR head SHA from gate (works for both pull_request and issue_comment)
ref: ${{ needs.gate.outputs.pr_head_sha }}
fetch-depth: 0
- name: Merge main into PR branch
if: github.event_name == 'issue_comment'
run: |
git config user.email "ci@github.com"
git config user.name "CI"
git remote add upstream ${{ github.server_url }}/${{ github.repository }}.git
git fetch upstream main
git merge upstream/main --no-edit
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "1.25.x"
cache-dependency-path: ./go.sum
- name: Verify Go toolchain
run: deploy/ci-e2e-openshift/verify-go-toolchain.sh
- name: Install tools (kubectl, oc, helm, make, jq, yq)
run: deploy/ci-e2e-openshift/install-tools.sh
- name: Verify cluster access
run: deploy/ci-e2e-openshift/verify-cluster-access.sh
- name: Verify correct cluster (vllm-d, not pok-prod)
run: deploy/ci-e2e-openshift/verify-correct-cluster.sh
- name: Check GPU availability
id: gpu-check
run: deploy/ci-e2e-openshift/check-gpu-availability.sh
- name: Post GPU status to PR
if: always() && needs.gate.outputs.pr_number != ''
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ needs.gate.outputs.pr_number }}
run: |
GPU_STATUS="${{ steps.gpu-check.outcome }}"
GPU_AVAIL="${{ steps.gpu-check.outputs.gpu_available }}"
TOTAL_GPUS="${{ steps.gpu-check.outputs.total_gpus }}"
ALLOCATED_GPUS="${{ steps.gpu-check.outputs.allocated_gpus }}"
AVAILABLE_GPUS="${{ steps.gpu-check.outputs.available_gpus }}"
TOTAL_CPU="${{ steps.gpu-check.outputs.total_cpu }}"
TOTAL_MEM_GI="${{ steps.gpu-check.outputs.total_mem_gi }}"
NODE_COUNT="${{ steps.gpu-check.outputs.node_count }}"
GPU_NODE_COUNT="${{ steps.gpu-check.outputs.gpu_node_count }}"
REQUIRED_GPUS="${{ steps.gpu-check.outputs.required_gpus }}"
RECOMMENDED_GPUS="${{ steps.gpu-check.outputs.recommended_gpus }}"
NL=$'\n'
TABLE="| Resource | Total | Allocated | Available |${NL}|----------|-------|-----------|----------|${NL}| GPUs | $TOTAL_GPUS | $ALLOCATED_GPUS | **$AVAILABLE_GPUS** |${NL}${NL}| Cluster | Value |${NL}|---------|-------|${NL}| Nodes | $NODE_COUNT ($GPU_NODE_COUNT with GPUs) |${NL}| Total CPU | ${TOTAL_CPU} cores |${NL}| Total Memory | ${TOTAL_MEM_GI} Gi |${NL}| GPUs required | $REQUIRED_GPUS (min) / $RECOMMENDED_GPUS (recommended) |"
if [ "$GPU_STATUS" = "failure" ]; then
HEADER="### GPU Pre-flight Check ❌"
MSG="**Insufficient GPUs** to run OpenShift E2E. Re-run with \`/retest\` (OpenShift E2E) when GPUs free up."
elif [ "$GPU_AVAIL" = "true" ]; then
HEADER="### GPU Pre-flight Check ✅"
MSG="GPUs are available for e2e-openshift tests. Proceeding with deployment."
else
HEADER="### GPU Pre-flight Check ⚠️"
MSG="Low GPU headroom — tests may fail during scale-up phases."
fi
BODY="${HEADER}${NL}${MSG}${NL}${NL}${TABLE}"
PAYLOAD=$(jq -n --arg body "$BODY" '{"body": $body}')
curl -s -X POST \
-H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/issues/$PR_NUMBER/comments" \
-d "$PAYLOAD"
- name: Get HF token from cluster secret
id: hf-token
run: deploy/ci-e2e-openshift/get-hf-token.sh
- name: Clean up resources for this PR
run: deploy/ci-e2e-openshift/cleanup-pr-resources.sh
- name: Deploy WVA and llm-d infrastructure
env:
# HF_TOKEN is inherited from GITHUB_ENV (set in 'Get HF token from cluster secret' step)
ENVIRONMENT: openshift
SCALER_BACKEND: keda
LLMD_NS: ${{ env.LLMD_NAMESPACE }}
WVA_NS: ${{ env.WVA_NAMESPACE }}
CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }}
MONITORING_NAMESPACE: openshift-user-workload-monitoring
WVA_METRICS_SECURE: "false"
ENABLE_SCALE_TO_ZERO: "true"
KV_SPARE_TRIGGER: "0.5"
QUEUE_SPARE_TRIGGER: "4.5"
VLLM_SVC_PORT: "8000"
ACCELERATOR_TYPE: ${{ env.ACCELERATOR_TYPE }}
GAIE_VERSION: ${{ env.GAIE_VERSION }}
run: deploy/ci-e2e-openshift/deploy-infrastructure.sh
- name: Label namespaces for OpenShift monitoring
run: deploy/ci-e2e-openshift/label-namespaces.sh
- name: Wait for infrastructure to be ready
run: deploy/ci-e2e-openshift/wait-infrastructure-ready.sh
- name: Verify deployment
run: deploy/ci-e2e-openshift/verify-deployment.sh
- name: Verify metrics pipeline
run: deploy/ci-e2e-openshift/verify-metrics-pipeline.sh
- name: Install Go dependencies
run: deploy/ci-e2e-openshift/install-go-dependencies.sh
- name: Run OpenShift E2E tests
env:
# Consolidated e2e test environment variables
ENVIRONMENT: openshift
# Temporary/ To do: real vLLM decode scheduling was flaky on shared GPU runners; tests use llm-d-inference-sim.
USE_SIMULATOR: "true"
SCALE_TO_ZERO_ENABLED: "true"
DEPLOY_LWS: "true"
WVA_NAMESPACE: ${{ env.WVA_NAMESPACE }}
MONITORING_NAMESPACE: openshift-user-workload-monitoring
LLMD_NAMESPACE: ${{ env.LLMD_NAMESPACE }}
# Legacy variables for backward compatibility (if needed by tests)
CONTROLLER_NAMESPACE: ${{ env.WVA_NAMESPACE }}
GATEWAY_NAME: optimized-baseline-inference-gateway-istio
# Pass WVA_RELEASE_NAME so test can filter for current run's resources
WVA_RELEASE_NAME: ${{ env.WVA_RELEASE_NAME }}
# Controller instance label must match what the controller was deployed with
CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }}
MODEL_ID: ${{ env.MODEL_ID }}
REQUEST_RATE: ${{ env.REQUEST_RATE }}
NUM_PROMPTS: ${{ env.NUM_PROMPTS }}
run: deploy/ci-e2e-openshift/run-e2e-tests.sh
- name: Cleanup infrastructure
# Cleanup on success or cancellation, but NOT on failure (preserve for debugging)
# Use SKIP_CLEANUP=true to keep resources after successful runs
if: (success() || cancelled()) && env.SKIP_CLEANUP != 'true'
run: deploy/ci-e2e-openshift/cleanup-infrastructure.sh
- name: Collect cluster diagnostics
if: always()
run: deploy/ci-e2e-openshift/collect-diagnostics.sh
- name: Upload cluster diagnostics
if: always()
uses: actions/upload-artifact@v7
with:
name: cluster-diagnostics-${{ github.run_id }}
path: /tmp/cluster-diagnostics/
retention-days: 7
- name: Scale down GPU workloads on failure
# On failure, scale down decode deployments to free GPUs while preserving
# other resources (VA, HPA, controller, gateway) for debugging
if: failure()
run: deploy/ci-e2e-openshift/scale-down-on-failure.sh
# Report status back to PR for issue_comment triggered runs
# This ensures fork PRs show the correct status after /ok-to-test runs complete
report-status:
runs-on: ubuntu-latest
needs: [gate, e2e-openshift]
# Run always (even on failure) but only for issue_comment events
if: always() && github.event_name == 'issue_comment' && needs.gate.outputs.should_run == 'true'
steps:
- name: Report status to PR
uses: actions/github-script@v7
with:
script: |
const prHeadSha = '${{ needs.gate.outputs.pr_head_sha }}';
const e2eResult = '${{ needs.e2e-openshift.result }}';
// Map job result to commit status
let state, description;
if (e2eResult === 'success') {
state = 'success';
description = 'E2E tests passed';
} else if (e2eResult === 'skipped') {
state = 'pending';
description = 'E2E tests skipped';
} else if (e2eResult === 'cancelled') {
state = 'failure';
description = 'E2E tests cancelled';
} else {
state = 'failure';
description = 'E2E tests failed';
}
console.log(`Reporting status to PR commit ${prHeadSha}: ${state} - ${description}`);
await github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha: prHeadSha,
state: state,
target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
description: description,
context: '${{ github.workflow }} / e2e (comment trigger)'
});
console.log('Status reported successfully');