Skip to content

Commit b10d07e

Browse files
arjunsuresharjunsuresh
authored
Merge pull request #331 from mlcommons/mlperf-inference
mlperf-inference to main (to run tests)
2 parents 9be5704 + 238325f commit b10d07e

File tree

34 files changed

+385
-131
lines changed

34 files changed

+385
-131
lines changed

.github/workflows/check-broken-links.md renamed to .github/workflows/check-broken-links.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
1-
name: Check .md README files for broken links
1+
name: "Check .md README files for broken links"
22

3-
on: [pull_request]
3+
on:
4+
push:
5+
branches:
6+
- master
47

58
jobs:
69
markdown-link-check:
710
runs-on: ubuntu-latest
811
# check out the latest version of the code
912
steps:
10-
- uses: actions/checkout@v3
13+
- uses: actions/checkout@v4
1114

1215
# Checks the status of hyperlinks in .md files in verbose mode
1316
- name: Check links

‎.github/workflows/code-review.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: OpenAI Code Review
22

33
on:
44
pull_request_target:
5-
types: [opened, synchronize]
5+
types: [opened]
66
paths:
77
- 'automation/**'
88
- 'script/**'
@@ -15,7 +15,7 @@ permissions:
1515
jobs:
1616
code_review:
1717
runs-on: ubuntu-latest
18-
if: github.repository_owner == 'gateoverflow' && github.event.pull_request.changed_files > 0
18+
if: github.repository_owner == 'gateoverflow_off' && github.event.pull_request.changed_files > 0
1919
steps:
2020
# Run code review via OpenAI
2121
# Step to run the OpenAI Code Review using the GATEOverflow action

‎.github/workflows/test-mlperf-inference-gptj.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ name: MLPerf inference GPT-J
55

66
on:
77
schedule:
8-
- cron: "1 1 * * */3"
8+
- cron: "1 2 * * *"
99

1010
jobs:
1111
build:
1212
if: github.repository_owner == 'gateoverflow'
13-
runs-on: [ self-hosted, linux, x64 ]
13+
runs-on: [ self-hosted, linux, x64, GO-spr ]
1414
strategy:
1515
fail-fast: false
1616
matrix:
@@ -24,7 +24,10 @@ jobs:
2424
source gh_action/bin/deactivate || python3 -m venv gh_action
2525
source gh_action/bin/activate
2626
export CM_REPOS=$HOME/GH_CM
27-
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
27+
python3 -m pip install cm4mlops
28+
cm pull repo
2829
- name: Test MLPerf Inference GPTJ
2930
run: |
3031
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
32+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
33+

‎.github/workflows/test-mlperf-inference-llama2.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3+
4+
name: MLPerf inference LLAMA 2 70B
5+
6+
on:
7+
schedule:
8+
- cron: "30 19 * * 4"
9+
10+
jobs:
11+
build_reference:
12+
if: github.repository_owner == 'gateoverflow'
13+
runs-on: [ self-hosted, GO-i9, linux, x64 ]
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
python-version: [ "3.12" ]
18+
backend: [ "pytorch" ]
19+
device: [ "cpu" ]
20+
21+
steps:
22+
- name: Install dependencies
23+
run: |
24+
source gh_action/bin/deactivate || python3 -m venv gh_action
25+
source gh_action/bin/activate
26+
export CM_REPOS=$HOME/GH_CM
27+
python3 -m pip install cm4mlops
28+
cm pull repo
29+
python3 -m pip install "huggingface_hub[cli]"
30+
huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
31+
- name: Test MLPerf Inference LLAMA 2 70B reference implementation
32+
run: |
33+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference.git --clean

‎.github/workflows/test-mlperf-inference-resnet50.yml‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
name: MLPerf inference ResNet50
55

66
on:
7-
pull_request:
7+
pull_request_target:
88
branches: [ "main", "dev", "mlperf-inference" ]
99
paths:
1010
- '.github/workflows/test-mlperf-inference-resnet50.yml'
@@ -28,9 +28,7 @@ jobs:
2828
- os: macos-latest
2929
backend: tf
3030
- os: windows-latest
31-
# MLPerf requires interaction when installing LLVM on Windows - that's why we excluded it here
32-
33-
31+
implementation: cpp
3432
steps:
3533
- uses: actions/checkout@v4
3634
- name: Set up Python ${{ matrix.python-version }}
@@ -41,6 +39,26 @@ jobs:
4139
run: |
4240
python3 -m pip install cmind
4341
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
44-
- name: Test MLPerf Inference ResNet50
42+
- name: Test MLPerf Inference ResNet50 (Windows)
43+
if: matrix.os == 'windows-latest'
4544
run: |
46-
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="cTuning" --hw_name=default --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
45+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_windows --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
46+
- name: Test MLPerf Inference ResNet50 (Linux/macOS)
47+
if: matrix.os != 'windows-latest'
48+
run: |
49+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
50+
- name: Push Results
51+
if: github.repository_owner == 'gateoverflow'
52+
env:
53+
USER: "GitHub Action"
54+
EMAIL: "[email protected]"
55+
run: |
56+
git config --global user.name "$USER"
57+
git config --global user.email "$EMAIL"
58+
git config --global credential.https://github.com.helper ""
59+
git config --global credential.https://github.com.helper "!gh auth git-credential"
60+
git config --global credential.https://gist.github.com.helper ""
61+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
62+
63+
cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}"
64+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action" --quiet

‎.github/workflows/test-mlperf-inference-sdxl.yaml‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
name: MLPerf inference SDXL
2-
2+
#off now as we have SCC24 test doing the same
33
on:
44
schedule:
55
- cron: "1 2 * * *"
66

77
jobs:
88
build_reference:
9-
if: github.repository_owner == 'gateoverflow'
9+
if: github.repository_owner == 'gateoverflow_off'
1010
runs-on: [ self-hosted, linux, x64 ]
1111
strategy:
1212
fail-fast: false
@@ -15,18 +15,17 @@ jobs:
1515
backend: [ "pytorch" ]
1616
precision: [ "float16" ]
1717
steps:
18-
- name: Install dependencies
18+
- name: Test MLPerf Inference SDXL Reference
1919
run: |
2020
source gh_action/bin/deactivate || python3 -m venv gh_action
2121
source gh_action/bin/activate
2222
export CM_REPOS=$HOME/GH_CM
23-
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
24-
- name: Test MLPerf Inference SDXL
25-
run: |
23+
python3 -m pip install cm4mlops
24+
cm pull repo
2625
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
2726
2827
build_nvidia:
29-
if: github.repository_owner == 'gateoverflow'
28+
if: github.repository_owner == 'gateoverflow_off'
3029
runs-on: [ self-hosted, linux, x64 ]
3130
strategy:
3231
fail-fast: false
@@ -36,12 +35,10 @@ jobs:
3635
precision: [ "float16" ]
3736
implementation: [ "nvidia" ]
3837
steps:
39-
- name: Install dependencies
38+
- name: Test MLPerf Inference SDXL Nvidia
4039
run: |
4140
source gh_action/bin/deactivate || python3 -m venv gh_action
4241
source gh_action/bin/activate
4342
export CM_REPOS=$HOME/GH_CM
44-
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
45-
- name: Test MLPerf Inference SDXL
46-
run: |
43+
cm pull repo
4744
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean

‎.github/workflows/test-scc24-sdxl.yaml‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1-
name: MLPerf inference SDXL
1+
name: MLPerf inference SDXL (SCC)
22

33
on:
44
schedule:
5-
- cron: "43 1 * * *"
5+
- cron: "1 3 * * *"
66

77
jobs:
88
build_reference:
99
if: github.repository_owner == 'gateoverflow'
10-
runs-on: [ self-hosted, linux, x64 ]
10+
runs-on: [ self-hosted, linux, x64, GO-spr ]
11+
env:
12+
CM_REPOS: $HOME/GH_CM
1113
strategy:
1214
fail-fast: false
1315
matrix:
@@ -16,23 +18,23 @@ jobs:
1618
precision: [ "float16" ]
1719
device: [ "cuda" ]
1820
steps:
19-
- name: Install dependencies
21+
- name: Test MLPerf Inference reference SDXL SCC
2022
run: |
21-
source gh_action/bin/deactivate || python3 -m venv gh_action
23+
if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
24+
python3 -m venv gh_action
2225
source gh_action/bin/activate
2326
export CM_REPOS=$HOME/GH_CM
24-
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
25-
- name: Test MLPerf Inference reference SDXL SCC
26-
env:
27-
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
28-
run: |
29-
cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --clean |
30-
cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons |
31-
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet
27+
pip install --upgrade cm4mlops
28+
pip install tabulate
29+
cm pull repo
30+
cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
31+
cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
32+
cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results
33+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions
3234
3335
build_nvidia:
3436
if: github.repository_owner == 'gateoverflow'
35-
runs-on: [ self-hosted, linux, x64 ]
37+
runs-on: [ self-hosted, linux, x64, GO-spr]
3638
strategy:
3739
fail-fast: false
3840
matrix:
@@ -41,16 +43,16 @@ jobs:
4143
precision: [ "float16" ]
4244
implementation: [ "nvidia" ]
4345
steps:
44-
- name: Install dependencies
46+
- name: Test MLPerf Inference NVIDIA SDXL SCC
4547
run: |
46-
source gh_action/bin/deactivate || python3 -m venv gh_action
48+
if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
49+
python3 -m venv gh_action
4750
source gh_action/bin/activate
4851
export CM_REPOS=$HOME/GH_CM
49-
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
50-
- name: Test MLPerf Inference NVIDIA SDXL SCC
51-
env:
52-
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
53-
run: |
54-
cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --clean |
55-
cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons |
56-
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet
52+
pip install --upgrade cm4mlops
53+
pip install tabulate
54+
cm pull repo
55+
cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --clean
56+
cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
57+
cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results
58+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions

‎README.md‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
## Unified and cross-platform CM interface for DevOps, MLOps and MLPerf
22

3-
[![arXiv](https://img.shields.io/badge/arXiv-2406.16791-b31b1b.svg)](https://arxiv.org/abs/2406.16791)
43
[![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md)
54
[![Python Version](https://img.shields.io/badge/python-3+-blue.svg)](https://github.com/mlcommons/ck/tree/master/cm/cmind)
65
[![Powered by CM](https://img.shields.io/badge/Powered_by-MLCommons%20CM-blue)](https://github.com/mlcommons/ck).
@@ -141,8 +140,7 @@ cm run script \
141140

142141
## CM concepts
143142

144-
* https://doi.org/10.5281/zenodo.8105339
145-
* https://arxiv.org/abs/2406.16791
143+
Check our [ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339).
146144

147145
## Authors
148146

‎script/app-image-corner-detection/_cm.json‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎script/app-mlperf-inference-ctuning-cpp-tflite/_cm.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@
120120
{
121121
"names": [
122122
"tensorflow",
123-
"tflite"
123+
"tflite"
124124
],
125125
"tags": "get,tensorflow,lib,_tflite"
126126
},

0 commit comments

Comments
 (0)