InferenceX/.github/workflows/e2e-tests.yml at 7164cdef8c5bd21e5b8369f9caea24e530e934b8 · SemiAnalysisAI/InferenceX · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
name: End-to-End Tests
run-name: e2e Test - ${{ github.event.inputs.generate-cli-command }}

# concurrency:
#     group: benchmark-lock
#     cancel-in-progress: false

on:
    workflow_dispatch:
        inputs:
            generate-cli-command:
                description: "Command passed to generate matrix script"
                required: true
                type: string

jobs:
    get-jobs:
        runs-on: ubuntu-latest
        outputs:
            search-space-config: ${{ steps.get-jobs.outputs.search-space-config }}
        steps:
            - name: Checkout code
              uses: actions/checkout@v4

            - id: get-jobs
              run: |
                  pip install pydantic
                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py ${{ inputs.generate-cli-command }})
                  echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT

    test-sweep:
        needs: get-jobs
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: ${{ inputs.generate-cli-command }}
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-jobs.outputs.search-space-config) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    calc-success-rate:
        needs: test-sweep
        if: ${{ always() }}
        runs-on: ubuntu-latest

        env:
            RESULTS_DIR: "results/"
            STATS_FILENAME: "run_stats"
            GITHUB_TOKEN: ${{ secrets.REPO_PAT }}

        steps:
            - uses: actions/checkout@v3
              with:
                  token: ${{ secrets.REPO_PAT }}
                  fetch-depth: 0

            - name: Download results artifacts
              uses: actions/download-artifact@v4
              with:
                  path: ${{ env.RESULTS_DIR }}
                  pattern: results_*

            - name: Install python dependencies
              run: pip install PyGithub

            - name: Calculate success rate
              run: python3 utils/calc_success_rate.py $STATS_FILENAME

            - uses: actions/upload-artifact@v4
              with:
                  name: "run-stats"
                  path: ${{ env.STATS_FILENAME }}.json