Skip to content

Commit 988eaed

Browse files
committed
Added validate-mlc-json mlc script
1 parent 3b82852 commit 988eaed

6 files changed

Lines changed: 128 additions & 20 deletions

File tree

.github/workflows/test-mlc-script-features.yml

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -286,15 +286,9 @@ jobs:
286286
- name: Test print_deps option
287287
shell: bash
288288
run: |
289-
# detect-cpu has deps on detect-os, print_deps should list them
290-
OUTPUT=$(mlcr detect,cpu --print_deps --quiet 2>&1)
291-
if echo "$OUTPUT" | grep -qi "detect.*os\|mlcr"; then
292-
echo "PASS: print_deps correctly listed dependency commands"
293-
else
294-
echo "WARNING: print_deps output may be empty if deps resolved from cache"
295-
fi
296-
# Ensure the command itself succeeds
297-
mlcr detect,cpu --print_deps --quiet
289+
# detect-cpu has deps on detect-os, verify deps resolved via validate script
290+
mlcr detect,cpu --print_deps --json --quiet > /tmp/mlc-printdeps-test.json 2>&1
291+
mlcr validate,mlc-json --json_file=/tmp/mlc-printdeps-test.json --expected_return=0 --dep_tags="detect,os" --quiet
298292
299293
- name: Test silent option
300294
shell: bash
@@ -539,14 +533,9 @@ jobs:
539533
- name: Test json output option
540534
shell: bash
541535
run: |
542-
# --json should produce JSON-formatted output
543-
OUTPUT=$(mlcr detect,os --json --quiet 2>&1)
544-
# Check if output contains JSON-like structure (curly braces)
545-
if echo "$OUTPUT" | grep -q '{'; then
546-
echo "PASS: --json produced JSON-formatted output"
547-
else
548-
echo "WARNING: --json output may not contain JSON"
549-
fi
536+
# --json should produce valid JSON with return code, env, and expected keys
537+
mlcr detect,os --json --quiet > /tmp/mlc-json-test.json 2>&1
538+
mlcr validate,mlc-json --json_file=/tmp/mlc-json-test.json --expected_return=0 --env_keys=MLC_HOST_OS_TYPE,MLC_HOST_OS_BITS --quiet
550539
551540
- name: Test j shorthand for json
552541
shell: bash

script/README.md

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# MLCommons Automation Scripts
22

3-
*Last updated: 2026-04-10 22:41:27*
3+
*Last updated: 2026-04-14 21:28:16*
44

55
This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations.
66

@@ -33,6 +33,7 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
3333
- [ROCm automation](#rocm-automation)
3434
- [Remote automation](#remote-automation)
3535
- [Reproducibility and artifact evaluation](#reproducibility-and-artifact-evaluation)
36+
- [Testing](#testing)
3637
- [Tests](#tests)
3738
- [TinyML automation](#tinyml-automation)
3839
- [Uncategorized](#uncategorized)
@@ -937,6 +938,12 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
937938
- get-ipol-src
938939
- Tags: `get`, `ipol`, `journal`, `src`, `ipol-src`
939940

941+
## Testing
942+
943+
- **[validate-mlc-json](validate-mlc-json/)**
944+
- validate-mlc-json
945+
- Tags: `validate`, `mlc-json`
946+
940947
## Tests
941948

942949
- **[run-python](run-python/)**
@@ -1109,8 +1116,8 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
11091116

11101117
## Statistics
11111118

1112-
- **Total Scripts**: 324
1113-
- **Categories**: 31
1119+
- **Total Scripts**: 325
1120+
- **Categories**: 32
11141121

11151122
## Usage
11161123

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import os
2+
import json
3+
import re
4+
5+
6+
def preprocess(i):
7+
8+
env = i['env']
9+
10+
json_file = env.get('MLC_VALIDATE_JSON_FILE', '')
11+
if not json_file:
12+
return {'return': 1, 'error': 'json_file is required (--json_file=<path>)'}
13+
14+
if not os.path.isfile(json_file):
15+
return {'return': 1, 'error': f'JSON file not found: {json_file}'}
16+
17+
with open(json_file, 'r') as f:
18+
raw = f.read()
19+
20+
# Extract JSON block from potentially mixed log+json output
21+
match = re.search(r'\{.*\}', raw, re.DOTALL)
22+
if not match:
23+
return {'return': 1, 'error': 'No JSON object found in file'}
24+
25+
try:
26+
data = json.loads(match.group())
27+
except json.JSONDecodeError as e:
28+
return {'return': 1, 'error': f'Invalid JSON: {e}'}
29+
30+
errors = []
31+
32+
# Check return code
33+
expected_return = env.get('MLC_VALIDATE_EXPECTED_RETURN', '')
34+
if expected_return != '':
35+
actual = data.get('return')
36+
if actual != int(expected_return):
37+
errors.append(f'Expected return={expected_return}, got {actual}')
38+
39+
# Check env keys exist
40+
env_keys = env.get('MLC_VALIDATE_ENV_KEYS', '')
41+
if env_keys:
42+
result_env = data.get('env', {})
43+
for key in env_keys.split(','):
44+
key = key.strip()
45+
if key and key not in result_env:
46+
errors.append(f'Expected env key "{key}" not found')
47+
48+
# Check env keys do NOT exist
49+
not_env_keys = env.get('MLC_VALIDATE_NOT_ENV_KEYS', '')
50+
if not_env_keys:
51+
result_env = data.get('env', {})
52+
for key in not_env_keys.split(','):
53+
key = key.strip()
54+
if key and key in result_env:
55+
errors.append(f'Env key "{key}" should not be present but was found')
56+
57+
# Check env key=value pairs (format: KEY=VALUE;KEY2=VALUE2)
58+
env_key_values = env.get('MLC_VALIDATE_ENV_KEY_VALUES', '')
59+
if env_key_values:
60+
result_env = data.get('env', {})
61+
for pair in env_key_values.split(';'):
62+
if '=' in pair:
63+
k, v = pair.split('=', 1)
64+
k = k.strip()
65+
v = v.strip()
66+
actual_v = result_env.get(k)
67+
if actual_v is None:
68+
errors.append(f'Expected env key "{k}" not found')
69+
elif str(actual_v) != v:
70+
errors.append(f'Expected env {k}="{v}", got "{actual_v}"')
71+
72+
# Check dep tags exist in deps list (semicolon-separated groups)
73+
dep_tags = env.get('MLC_VALIDATE_DEP_TAGS', '')
74+
if dep_tags:
75+
deps = data.get('deps', [])
76+
all_dep_tags = [d.get('tags', '') for d in deps]
77+
for tag_set in dep_tags.split(';'):
78+
tag_set = tag_set.strip()
79+
if tag_set and not any(tag_set in t for t in all_dep_tags):
80+
errors.append(f'Expected dep tags "{tag_set}" not found in deps: {all_dep_tags}')
81+
82+
if errors:
83+
error_msg = 'JSON validation failed:\n' + '\n'.join(f' - {e}' for e in errors)
84+
return {'return': 1, 'error': error_msg}
85+
86+
print('JSON validation PASSED')
87+
return {'return': 0}
88+
89+
90+
def postprocess(i):
91+
return {'return': 0}

script/validate-mlc-json/meta.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
alias: validate-mlc-json
2+
automation_alias: script
3+
automation_uid: 5b4e0237da074764
4+
category: Testing
5+
input_mapping:
6+
json_file: MLC_VALIDATE_JSON_FILE
7+
expected_return: MLC_VALIDATE_EXPECTED_RETURN
8+
env_keys: MLC_VALIDATE_ENV_KEYS
9+
env_key_values: MLC_VALIDATE_ENV_KEY_VALUES
10+
dep_tags: MLC_VALIDATE_DEP_TAGS
11+
not_env_keys: MLC_VALIDATE_NOT_ENV_KEYS
12+
tags:
13+
- validate
14+
- mlc-json
15+
uid: cccc1111dddd2222

script/validate-mlc-json/run.bat

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
@echo off
2+
rem No-op: all validation done in customize.py
3+
exit /b 0

script/validate-mlc-json/run.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
# No-op: all validation done in customize.py
3+
exit 0

0 commit comments

Comments
 (0)