Fix automatic image tag selection by version #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - PR Benchmark Run | |
| on: | |
| pull_request: | |
| jobs: | |
| # -- Unit tests (no cluster needed) -- | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v7.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install llmdbenchmark | |
| run: | | |
| ./install.sh -y 2>&1 | |
| pip install pytest | |
| - name: Run unit tests | |
| run: python -m pytest tests/ -x -q | |
| # -- Standalone deployment (own Kind cluster) -- | |
| benchmark-standalone: | |
| name: Benchmark - Standalone (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: cicd | |
| standup_method: standalone | |
| standup_scenario: kind | |
| decode_pods: auto | |
| prefill_pods: auto | |
| accelerator_type: X | |
| backend_type: X | |
| infra_provider: X | |
| connector: X | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: none | |
| monitoring_enabled: true | |
| dry_run: false | |
| verbose: false | |
| harness: vllm-benchmark | |
| workload: sanity_random.yaml | |
| # harness: nop | |
| # workload: nop.yaml | |
| secrets: inherit | |
| # -- FMA deployment (own Kind cluster) -- | |
| benchmark-fma: | |
| name: Benchmark - FMA (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: cicd | |
| standup_method: fma | |
| standup_scenario: kind | |
| decode_pods: auto | |
| prefill_pods: auto | |
| accelerator_type: X | |
| backend_type: X | |
| infra_provider: X | |
| connector: X | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: none | |
| monitoring_enabled: true | |
| dry_run: false | |
| verbose: false | |
| harness: nop | |
| workload: nop.yaml | |
| secrets: inherit | |
| benchmark-modelservice-istio: | |
| name: Benchmark - Modelservice / istio (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: cicd | |
| standup_method: modelservice | |
| standup_scenario: kind | |
| decode_pods: auto | |
| prefill_pods: auto | |
| accelerator_type: X | |
| backend_type: X | |
| infra_provider: X | |
| connector: X | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: istio | |
| monitoring_enabled: true | |
| dry_run: false | |
| verbose: false | |
| harness: inference-perf | |
| workload: sanity_random.yaml | |
| secrets: inherit | |
| benchmark-modelservice-agentgateway: | |
| name: Benchmark - Modelservice / agentgateway (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: cicd | |
| standup_method: modelservice | |
| standup_scenario: kind | |
| decode_pods: auto | |
| prefill_pods: auto | |
| accelerator_type: X | |
| backend_type: X | |
| infra_provider: X | |
| connector: X | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: agentgateway | |
| monitoring_enabled: true | |
| dry_run: false | |
| verbose: false | |
| harness: guidellm | |
| workload: sanity_random.yaml | |
| secrets: inherit | |
| benchmark-modelservice-epponly: | |
| name: Benchmark - Modelservice / epponly (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: cicd | |
| standup_method: modelservice | |
| standup_scenario: kind | |
| decode_pods: auto | |
| prefill_pods: auto | |
| accelerator_type: X | |
| backend_type: X | |
| infra_provider: X | |
| connector: X | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: epponly | |
| monitoring_enabled: true | |
| dry_run: false | |
| verbose: false | |
| harness: guidellm | |
| workload: sanity_random.yaml | |
| secrets: inherit | |
| benchmark-kustomize-optimized-baseline: | |
| name: Benchmark - kustomize / epponly / optimized-baseline (Kind) | |
| uses: llm-d/llm-d-benchmark/.github/workflows/reusable-ci-nightly-benchmark.yaml@main | |
| with: | |
| scenario_dir: guides | |
| standup_method: kustomize | |
| standup_scenario: optimized-baseline | |
| decode_pods: 2 | |
| prefill_pods: auto | |
| accelerator_type: gpu | |
| backend_type: vllm | |
| infra_provider: base | |
| connector: | |
| cluster_namespace: llmdbenchcicd-sa-${{ github.event.number }} | |
| helm_release: lmdbenchcicdr | |
| workspace_dir: tmp/cicd-sa-${{ github.event.number }} | |
| bucket_project: llm-d-scale | |
| bucket_provider: none | |
| bucket_path: /tmp/llmdbenchcicd | |
| gateway_class: epponly | |
| monitoring_enabled: true | |
| dry_run: true | |
| verbose: false | |
| harness: inference-perf | |
| workload: sanity_random.yaml | |
| secrets: inherit |