llm-d-benchmark/.github/workflows/ci-nighly-benchmark-ocp.yaml at 0c834dc5a97160d6575b20593fa1a4b45172ebba · kalantar/llm-d-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
name: CI - Nightly Run Benchmark

on:
  workflow_dispatch:
    inputs:
      input_dir:
        description: 'Input directory for benchmark results'
        required: false
        default: '/tmp/cicd/analysis'
      output_dir:
        description: 'Output directory name (S3 prefix and artifact name)'
        required: false
        default: ''

#  push:
#    branches:
#      - main

  schedule:
    - cron: '0 0 * * *'  # Daily at midnight UTC

jobs:
  run-benchmark:
    name: Benchmark Test
    runs-on: [k8s-util]
    timeout-minutes: 240

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Display OS used
        run: |
          cat /etc/*os-*
        shell: bash

      - name: Set input and output directory environment variables
        run: |
          DEFAULT_INPUT_DIR=/tmp/cicd/analysis
          INPUT_DIR="${{ github.event.inputs.input_dir }}"
          if [ -z "$INPUT_DIR" ]; then
            INPUT_DIR="$DEFAULT_INPUT_DIR"
          fi
          echo "INPUT_DIR=$INPUT_DIR" >> $GITHUB_ENV

          if [ -z "${{ github.event.inputs.output_dir }}" ]; then
            timestamp=$(date -u +%Y%m%dT%H%M%SZ)
            echo "OUTPUT_DIR=benchmark-results-${timestamp}" >> $GITHUB_ENV
            echo "Using generated output dir: benchmark-results-${timestamp}"
          else
            echo "OUTPUT_DIR=${{ github.event.inputs.output_dir }}" >> $GITHUB_ENV
            echo "Using provided output dir: ${{ github.event.inputs.output_dir }}"
          fi

      - name: Set up kubeconfig from secret
        run: |
          mkdir -p ~/.kube
          echo "${{ secrets.KUBECONFIG_DATA }}" | base64 -d > ~/.kube/config
          chmod 600 ~/.kube/config
        shell: bash

      - name: Run install_deps.sh
        run: |
          sudo apt-get update
          ./setup/install_deps.sh
        shell: bash

      - name: Install config explorer dependencies
        run: pip install ./config_explorer
        shell: bash

      - name: Cleanup target cloud (modelservice)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c ocp_L40_fb -t modelservice -d

      - name: Cleanup target cloud (standalone)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c ocp_L40_fb -t standalone -d

      - name: Standup target cloud (standalone)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/standup.sh -c ocp_L40_fb -t standalone

      - name: Run benchmark (standalone, inference-perf)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/run.sh -c ocp_L40_fb -t standalone

      - name: Run benchmark (standalone, fmperf)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/run.sh -c ocp_L40_fb -t standalone -l fmperf -w sanity_short-input

      - name: Run benchmark (standalone, guidellm)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/run.sh -c ocp_L40_fb -t standalone -l guidellm -w sanity_concurrent

      - name: Run benchmark (standalone, vllm-benchmark)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/run.sh -c ocp_L40_fb -t standalone -l vllm-benchmark

      - name: Cleanup target cloud (standalone)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c ocp_L40_fb -t standalone -d

      - name: E2E target cloud (modelservice, inference-perf)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/e2e.sh -c ocp_L40_fb -t modelservice --deep

      - name: E2E target cloud (modelservice, fmperf)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/e2e.sh -c ocp_L40_fb -t modelservice --deep -l fmperf -w sanity_short-input.yaml

      - name: E2E target cloud (modelservice, guidellm)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/e2e.sh -c ocp_L40_fb -t modelservice --deep -l guidellm -w sanity_concurrent.yaml


      - name: E2E target cloud (modelservice, vllm-benchmark)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/e2e.sh -c ocp_L40_fb -t modelservice --deep -l vllm-benchmark


      - name: Install AWS CLI
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install
          aws --version

      - name: Upload results to IBM COS
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          aws configure set default.s3.signature_version s3v4
          aws s3 cp "$INPUT_DIR" "s3://${{ secrets.COS_BUCKET_NAME }}/$OUTPUT_DIR/" \
            --recursive --endpoint-url ${{ secrets.COS_ENDPOINT_URL }} || true

      - name: Archive benchmark results as GitHub artifact
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.OUTPUT_DIR }}
          path: ${{ env.INPUT_DIR }}
          retention-days: 14