Skip to content

Commit 5858805

Browse files
authored
Merge pull request #378 from mlcommons/dev
Sync Dev
2 parents e08bc8c + 086a7a5 commit 5858805

File tree

149 files changed

+1151
-819
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

149 files changed

+1151
-819
lines changed

.github/workflows/run-tests-on-modified-meta.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ jobs:
3939
process_modified_files:
4040
runs-on: ubuntu-latest
4141
needs: get_modified_files
42+
if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != ''
4243
strategy:
4344
fail-fast: false
4445
matrix:

.github/workflows/test-amd-mlperf-inference-implementations.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ jobs:
1313
matrix:
1414
python-version: [ "3.12" ]
1515
model: [ "llama2-70b-99.9" ]
16+
1617
steps:
1718
- name: Test MLPerf Inference AMD (build only) ${{ matrix.model }}
1819
run: |

.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,9 @@ jobs:
4343
fail-fast: false
4444
matrix:
4545
python-version: [ "3.12" ]
46-
llvm-version: [ "15.0.6", "16.0.4", "17.0.6" ]
46+
compiler-string: [ "--adr.compiler.tags=gcc", "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes", "--adr.compiler.tags=llvm --adr.compiler.version=17.0.6" ]
4747
os: [ubuntu-latest, windows-latest, macos-latest]
4848
exclude:
49-
- llvm-version: "15.0.6"
50-
- llvm-version: "16.0.4"
5149
- os: windows-latest
5250
- os: macos-latest
5351

@@ -63,16 +61,15 @@ jobs:
6361
- name: Pull MLOps repository
6462
run: |
6563
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
66-
mlcr --quiet --tags=get,sys-utils-cm
67-
mlcr --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }}
64+
mlcr --quiet --tags=get,sys-utils-mlc
6865
- name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
6966
if: matrix.os == 'windows-latest'
7067
run: |
7168
mlcr app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet
7269
- name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
7370
if: matrix.os != 'windows-latest'
7471
run: |
75-
mlcr app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet
72+
mlcr app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet ${{ matrix.compiler-string }}
7673
- name: Randomly Execute Step
7774
id: random-check
7875
run: |

.github/workflows/test-mlperf-inference-resnet50-closed-division.yml

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ jobs:
5555
- uses: actions/checkout@v4
5656
with:
5757
fetch-depth: 0
58-
5958
- name: Set up Python ${{ matrix.python-version }}
6059
uses: actions/setup-python@v3
6160
with:
@@ -64,7 +63,15 @@ jobs:
6463
if: matrix.os == 'windows-latest'
6564
run: |
6665
git config --system core.longpaths true
67-
66+
- name: Export MLC_REPOS in Linux or Mac
67+
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
68+
run: |
69+
echo "MLC_REPOS=$HOME/gh action/mlc" >> $GITHUB_ENV
70+
- name: Export MLC_REPOS in Windows
71+
if: matrix.os == 'windows-latest'
72+
run: |
73+
$mlcrepos = "${env:USERPROFILE}\gh action\mlc"
74+
"MLC_REPOS=$mlcRepos" | Out-File -FilePath $env:GITHUB_ENV -Append
6875
- name: Install mlcflow
6976
run: |
7077
pip install mlcflow
@@ -77,12 +84,12 @@ jobs:
7784
- name: Test MLPerf Inference ResNet50 (Windows)
7885
if: matrix.os == 'windows-latest'
7986
run: |
80-
mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid
87+
mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference,_branch.patch-34
8188
8289
- name: Test MLPerf Inference ResNet50 Offline(Linux/macOS)
8390
if: matrix.os != 'windows-latest'
8491
run: |
85-
mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid
92+
mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference,_branch.patch-34
8693
8794
# Step for Linux/MacOS
8895
- name: Randomly Execute Step (Linux/MacOS)

.github/workflows/test-mlperf-inference-retinanet.yml

Lines changed: 4 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -9,34 +9,8 @@ on:
99
- '!**.md'
1010

1111
jobs:
12-
fetch-secret:
13-
runs-on: ubuntu-latest
14-
outputs:
15-
encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }}
16-
steps:
17-
- name: Load secret
18-
id: op-load-secret
19-
uses: 1password/load-secrets-action@v2
20-
with:
21-
export-env: false
22-
env:
23-
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
24-
PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential
25-
26-
- name: Encrypt secret
27-
id: encrypt-secret
28-
env:
29-
ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }}
30-
run: |
31-
# AES-256 encrypt
32-
encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \
33-
openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \
34-
-pass pass:"$ENCRYPTION_KEY" -base64 -A)
35-
36-
echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT
3712

3813
mlc-run:
39-
needs: [fetch-secret]
4014
runs-on: ${{ matrix.os }}
4115
strategy:
4216
fail-fast: false
@@ -45,13 +19,16 @@ jobs:
4519
python-version: [ "3.12" ]
4620
backend: [ "onnxruntime", "pytorch" ]
4721
implementation: [ "python", "cpp" ]
22+
compiler-string: [ "", "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes" ]
4823
exclude:
4924
- backend: pytorch
5025
implementation: cpp
5126
- os: windows-latest
5227
implementation: cpp
5328
- os: macos-latest
5429
implementation: cpp
30+
- implementation: python
31+
compiler-string: "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes"
5532

5633
steps:
5734
- uses: actions/checkout@v3
@@ -78,57 +55,4 @@ jobs:
7855
- name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }}
7956
if: matrix.os != 'windows-latest'
8057
run: |
81-
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1
82-
83-
# Step for Linux/MacOS
84-
- name: Randomly Execute Step (Linux/MacOS)
85-
if: runner.os != 'Windows'
86-
run: |
87-
RANDOM_NUMBER=$((RANDOM % 10))
88-
echo "Random number is $RANDOM_NUMBER"
89-
if [ "$RANDOM_NUMBER" -eq 0 ]; then
90-
echo "run_step=true" >> $GITHUB_ENV
91-
else
92-
echo "run_step=false" >> $GITHUB_ENV
93-
fi
94-
95-
# Step for Windows
96-
- name: Randomly Execute Step (Windows)
97-
if: runner.os == 'Windows'
98-
run: |
99-
$RANDOM_NUMBER = Get-Random -Maximum 10
100-
Write-Host "Random number is $RANDOM_NUMBER"
101-
if ($RANDOM_NUMBER -eq 0) {
102-
Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append
103-
} else {
104-
Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append
105-
}
106-
107-
- name: Decrypt secret
108-
id: decrypt-secret
109-
shell: bash
110-
env:
111-
ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }}
112-
encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }}
113-
run: |
114-
echo "Running on OS: ${{ matrix.os }}"
115-
116-
# Decrypt
117-
decrypted=$(echo "$encrypted_secret" | \
118-
openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \
119-
-pass pass:"$ENCRYPTION_KEY" -base64 -A)
120-
121-
echo "::add-mask::$decrypted"
122-
echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT
123-
- name: Push Results
124-
env:
125-
GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }}
126-
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
127-
run: |
128-
git config --global user.name "mlcommons-bot"
129-
git config --global user.email "[email protected]"
130-
git config --global credential.https://github.com.helper ""
131-
git config --global credential.https://github.com.helper "!gh auth git-credential"
132-
git config --global credential.https://gist.github.com.helper ""
133-
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
134-
mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
58+
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1 ${{ matrix.compiler-string }}

.github/workflows/test-mlperf-inference-rgat.yml

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ on:
99
- '!**.md'
1010

1111
jobs:
12+
1213
rgat-inference-run:
1314
name: ${{ matrix.os }} - ${{ matrix.backend }} - ${{ matrix.implementation }}
14-
needs: [fetch-secret]
1515
runs-on: ${{ matrix.os }}
1616
strategy:
1717
fail-fast: false
@@ -38,24 +38,3 @@ jobs:
3838
run: |
3939
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1
4040
41-
- name: Load secret
42-
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
43-
id: op-load-secret
44-
uses: 1password/load-secrets-action@v2
45-
with:
46-
export-env: false
47-
env:
48-
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
49-
PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential
50-
51-
- name: Push Results
52-
env:
53-
GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }}
54-
run: |
55-
git config --global user.name "mlcommons-bot"
56-
git config --global user.email "[email protected]"
57-
git config --global credential.https://github.com.helper ""
58-
git config --global credential.https://github.com.helper "!gh auth git-credential"
59-
git config --global credential.https://gist.github.com.helper ""
60-
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
61-
mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

automation/script/module.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3188,7 +3188,7 @@ def _update_variation_meta_with_dynamic_suffix(
31883188
item_value[i] = l_item.replace(
31893189
"#", variation_tag_dynamic_suffix)
31903190
else:
3191-
value[item] = value[item].replace(
3191+
value[item] = str(value[item]).replace(
31923192
"#", variation_tag_dynamic_suffix)
31933193

31943194
else: # scalar value, never used?
@@ -4469,6 +4469,13 @@ def docker(self, i):
44694469
from script.docker import docker_run
44704470
return docker_run(self, i)
44714471

4472+
############################################################
4473+
# portion for experiment action.
4474+
# as of now, the experiment action directly calls the run action.
4475+
# in the future, we will add more functionality to the experiment action.
4476+
def experiment(self, i):
4477+
return self.run(i)
4478+
44724479
##########################################################################
44734480

44744481
def _available_variations(self, i):

script/app-image-classification-onnx-py/customize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ def postprocess(i):
1919
os_info = i['os_info']
2020
env = i['env']
2121
state = i['state']
22-
2322
automation = i['automation']
2423
logger = automation.action_object.logger
2524

@@ -48,6 +47,7 @@ def postprocess(i):
4847
json.dump(data, f, ensure_ascii=False, indent=4)
4948
except Exception as e:
5049
logger.warning('CM warning: {}'.format(e))
50+
logger.warning('CM warning: {}'.format(e))
5151

5252
try:
5353
import yaml

script/app-image-corner-detection/customize.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def preprocess(i):
3838
def postprocess(i):
3939

4040
env = i['env']
41-
print(env['MLC_OUTPUT'] + " generated in " + env['MLC_RUN_DIR'])
41+
logger = i['automation'].logger
42+
logger.info(env['MLC_OUTPUT'] + " generated in " + env['MLC_RUN_DIR'])
4243

4344
return {'return': 0}

script/app-loadgen-generic-python/customize.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ def preprocess(i):
1111

1212
env = i['env']
1313

14+
logger = i['automation'].logger
15+
1416
if 'MLC_ML_MODEL_FILE_WITH_PATH' not in env:
1517
return {
1618
'return': 1, 'error': 'Please select a variation specifying the model to run'}
@@ -87,9 +89,9 @@ def preprocess(i):
8789

8890
env['MLC_RUN_OPTS'] = run_opts
8991

90-
print('')
91-
print('Assembled flags: {}'.format(run_opts))
92-
print('')
92+
logger.info('')
93+
logger.info('Assembled flags: {}'.format(run_opts))
94+
logger.info('')
9395

9496
return {'return': 0}
9597

0 commit comments

Comments
 (0)