llm-d-benchmark/.github/workflows/ci-nightly-benchmark-cks-fma.yaml at main · hubunt/llm-d-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
name: CI - Nightly Run Benchmark on CKS ("FMA" standup method)

on:
  workflow_dispatch:
    inputs:
      decode_pods:
        description: 'Number of decode (vllm "standalone") pods (default "auto", means what was configured on the scenario)'
        required: false
        type: 'string'
        default: 'auto'
      prefill_pods:
        description: 'Number of prefill pods (default "auto", means what was configured on the scenario)'
        required: false
        type: 'string'
        default: 'auto'
      cluster_namespace:
        description: 'Cluster namespace'
        required: false
        default: 'llmdbenchcicdfns-cks'
      gateway_class:
        description: 'Class of gateway used ("istio", "agentgateway", "epponly", "none")'
        required: false
        default: 'none'
      monitoring_enabled:
        description: 'Enabled monitoring ("true", "false")'
        required: true
        type: 'string'
        default: 'false'
      dry_run:
        description: 'Execute workflow in "dry-run" mode ("true", "false")'
        required: false
        default: 'false'
      verbose:
        description: 'Execute workflow in "verbose" mode ("true", "false")'
        required: false
        default: 'false'
        type: 'string'
      harness:
        description: 'Harness to be used during "run" operation ("inference-perf", "guidellm", "inferencemax", "vllm-benchmark", "nop")'
        required: false
        default: 'nop'
        type: 'string'
      workload:
        description: 'Workload profile to be used during "run" operation (check list under "workload/profiles")'
        required: false
        default: 'nop.yaml'
        type: 'string'
      cleanup:
        description: 'Cleanup the llm-d stack stood up by this workflow'
        required: false
        default: 'true'
        type: string

#  push:
#    branches:
#      - main

  schedule:
    - cron: '0 0 * * *'  # Daily at midnight UTC

jobs:
  nightly:
    uses: llm-d/llm-d-infra/.github/workflows/reusable-ci-nightly-benchmark.yaml@main
    with:
      scenario_dir: ${{ inputs.scenario_dir || 'cicd' }}
      standup_method: ${{ inputs.standup_method || 'fma' }}
      standup_scenario: ${{ inputs.standup_scenario || 'cks' }}
      decode_pods: ${{ inputs.decode_pods || 'auto' }}
      prefill_pods: ${{ inputs.prefill_pods || 'auto' }}
      accelerator_type: ${{ inputs.accelerator_type || 'gpu' }}
      backend_type: ${{ inputs.backend_type || 'vllm' }}
      infra_provider: ${{ inputs.infra_provider || 'coreweave' }}
      connector: ${{ inputs.connector || '' }}
      cluster_namespace: ${{ inputs.cluster_namespace || 'llmdbenchcicdfns-cks' }}
      helm_release: ${{ inputs.helm_release || 'lmdbenchcicdr-cks' }}
      workspace_dir: ${{ inputs.workspace_dir || '/tmp/llmdbenchcicdf-cks' }}
      bucket_project: ${{ inputs.bucket_project || 'llm-d-scale' }}
      bucket_provider: ${{ inputs.bucket_provider || 'gcs' }}
      bucket_path: ${{ inputs.bucket_path || 'llm-d-benchmarks/regressions' }}
      gateway_class: ${{ inputs.gateway_class || 'none' }}
      monitoring_enabled: ${{ inputs.monitoring_enabled || 'false' }}
      dry_run: ${{ inputs.dry_run || 'false' }}
      verbose: ${{ inputs.verbose || 'false' }}
      harness: ${{ inputs.harness || 'nop' }}
      workload: ${{ inputs.workload || 'nop.yaml' }}
      cleanup: ${{ inputs.cleanup || 'true' }}
    secrets: inherit