Skip to content

Commit 4776ec6

Browse files
committed
Added apptainer scripts, fixes mlcommons#484
1 parent 8e15156 commit 4776ec6

12 files changed

Lines changed: 1039 additions & 4 deletions

File tree

script/README.md

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# MLCommons Automation Scripts
22

3-
*Last updated: 2026-04-15 23:33:19*
3+
*Last updated: 2026-04-16 01:59:33*
44

55
This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations.
66

@@ -14,6 +14,7 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
1414
- [CUDA automation](#cuda-automation)
1515
- [Cloud automation](#cloud-automation)
1616
- [Compiler automation](#compiler-automation)
17+
- [Container automation](#container-automation)
1718
- [Dashboard automation](#dashboard-automation)
1819
- [Detection or installation of tools and artifacts](#detection-or-installation-of-tools-and-artifacts)
1920
- [DevOps automation](#devops-automation)
@@ -404,6 +405,18 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
404405
- Build vllm from sources
405406
- Tags: `install-vllm-from-src`
406407

408+
## Container automation
409+
410+
- **[build-apptainer-image](build-apptainer-image/)**
411+
- build-apptainer-image
412+
- Tags: `build`, `apptainer`, `image`, `apptainer-image`
413+
- **[build-apptainerfile](build-apptainerfile/)**
414+
- build-apptainerfile
415+
- Tags: `build`, `apptainerfile`, `apptainer`, `definition`
416+
- **[run-apptainer-container](run-apptainer-container/)**
417+
- run-apptainer-container
418+
- Tags: `run`, `apptainer`, `container`
419+
407420
## Dashboard automation
408421

409422
- **[publish-results-to-dashboard](publish-results-to-dashboard/)**
@@ -1116,8 +1129,8 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
11161129

11171130
## Statistics
11181131

1119-
- **Total Scripts**: 325
1120-
- **Categories**: 32
1132+
- **Total Scripts**: 328
1133+
- **Categories**: 33
11211134

11221135
## Usage
11231136

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
from mlc import utils
2+
import os
3+
from utils import *
4+
5+
6+
def preprocess(i):
7+
8+
os_info = i['os_info']
9+
env = i['env']
10+
logger = i['automation'].logger
11+
12+
if os_info['platform'] == 'windows':
13+
return {
14+
'return': 1,
15+
'error': 'Apptainer image building is not supported on Windows.'}
16+
17+
def_file_path = env.get('MLC_APPTAINERFILE_WITH_PATH', '')
18+
if def_file_path != '' and os.path.exists(def_file_path):
19+
build_def_file = False
20+
env['MLC_BUILD_APPTAINERFILE'] = "no"
21+
else:
22+
build_def_file = True
23+
env['MLC_BUILD_APPTAINERFILE'] = "yes"
24+
25+
# Determine image name
26+
image_name = env.get('MLC_APPTAINER_IMAGE_NAME', '')
27+
if image_name == '':
28+
image_name = "mlc-script-" + \
29+
env.get('MLC_APPTAINER_RUN_SCRIPT_TAGS', 'default').replace(
30+
',', '-').replace('_', '-')
31+
env['MLC_APPTAINER_IMAGE_NAME'] = image_name.lower()
32+
33+
image_tag = env.get('MLC_APPTAINER_IMAGE_TAG', 'latest')
34+
env['MLC_APPTAINER_IMAGE_TAG'] = image_tag
35+
36+
# SIF output path
37+
sif_name = f"{env['MLC_APPTAINER_IMAGE_NAME']}_{image_tag}.sif"
38+
sif_path = os.path.join(os.getcwd(), sif_name)
39+
env['MLC_APPTAINER_SIF_PATH'] = sif_path
40+
41+
# Build command options
42+
build_opts = ''
43+
44+
if is_true(env.get('MLC_APPTAINER_FAKEROOT', '')):
45+
build_opts += ' --fakeroot'
46+
47+
if is_true(env.get('MLC_APPTAINER_BUILD_SANDBOX', '')):
48+
# Build sandbox instead of SIF
49+
sandbox_path = sif_path.replace('.sif', '_sandbox')
50+
env['MLC_APPTAINER_SIF_PATH'] = sandbox_path
51+
build_opts += ' --sandbox'
52+
53+
if is_true(env.get('MLC_APPTAINER_BUILD_FORCE', '')):
54+
build_opts += ' --force'
55+
56+
if is_true(env.get('MLC_APPTAINER_BUILD_NOTEST', '')):
57+
build_opts += ' --notest'
58+
59+
if build_def_file:
60+
def_file_ref = "${MLC_APPTAINERFILE_WITH_PATH}"
61+
else:
62+
def_file_ref = def_file_path
63+
64+
output_path = env['MLC_APPTAINER_SIF_PATH']
65+
66+
CMD = f"apptainer build{build_opts} {output_path} {def_file_ref}"
67+
68+
logger.info('')
69+
logger.info(f'Apptainer build command:')
70+
logger.info(f' {CMD}')
71+
logger.info('')
72+
73+
env['MLC_APPTAINER_BUILD_CMD'] = CMD
74+
75+
# Save build script
76+
with open('apptainer-build.sh', 'w') as f:
77+
f.write('#!/bin/bash\n')
78+
f.write(CMD + '\n')
79+
80+
return {'return': 0}
81+
82+
83+
def postprocess(i):
84+
85+
env = i['env']
86+
logger = i['automation'].logger
87+
88+
sif_path = env.get('MLC_APPTAINER_SIF_PATH', '')
89+
if sif_path and os.path.exists(sif_path):
90+
logger.info(f'Apptainer image built: {sif_path}')
91+
else:
92+
logger.info(f'Apptainer image expected at: {sif_path}')
93+
94+
return {'return': 0}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
alias: build-apptainer-image
2+
uid: ec9e72fb6df8c7a6
3+
automation_alias: script
4+
automation_uid: 5b4e0237da074764
5+
cache: false
6+
category: Container automation
7+
tags:
8+
- build
9+
- apptainer
10+
- image
11+
- apptainer-image
12+
default_env:
13+
MLC_APPTAINER_IMAGE_REPO: local
14+
MLC_APPTAINER_IMAGE_TAG: latest
15+
input_mapping:
16+
def_file: MLC_APPTAINERFILE_WITH_PATH
17+
image_name: MLC_APPTAINER_IMAGE_NAME
18+
image_repo: MLC_APPTAINER_IMAGE_REPO
19+
image_tag: MLC_APPTAINER_IMAGE_TAG
20+
fakeroot: MLC_APPTAINER_FAKEROOT
21+
sandbox: MLC_APPTAINER_BUILD_SANDBOX
22+
force: MLC_APPTAINER_BUILD_FORCE
23+
os: MLC_APPTAINER_OS
24+
os_version: MLC_APPTAINER_OS_VERSION
25+
docker_os: MLC_APPTAINER_OS
26+
docker_os_version: MLC_APPTAINER_OS_VERSION
27+
script_tags: MLC_APPTAINER_RUN_SCRIPT_TAGS
28+
notest: MLC_APPTAINER_BUILD_NOTEST
29+
new_env_keys:
30+
- MLC_APPTAINER_*
31+
deps:
32+
- tags: get-apptainer
33+
prehook_deps:
34+
- enable_if_env:
35+
MLC_BUILD_APPTAINERFILE:
36+
- 'yes'
37+
tags: build,apptainerfile
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
if [ -f "${MLC_APPTAINERFILE_WITH_PATH}" ]; then
4+
5+
eval "${MLC_APPTAINER_BUILD_CMD}"
6+
test $? -eq 0 || exit 1
7+
8+
echo ""
9+
fi
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
{
2+
"python-packages": [
3+
"wheel",
4+
"mlcflow",
5+
"requests",
6+
"giturlparse",
7+
"tabulate"
8+
],
9+
"PYTHON": "python3",
10+
"distros": {
11+
"ubuntu": {
12+
"package-manager-update-cmd": "apt-get update -y",
13+
"package-manager-get-cmd": "apt-get install -y",
14+
"packages": [
15+
"python3",
16+
"python3-pip",
17+
"git",
18+
"sudo",
19+
"wget",
20+
"python3-venv",
21+
"unzip",
22+
"ca-certificates"
23+
],
24+
"versions": {
25+
"20.04": {
26+
"FROM": "docker://ubuntu:20.04"
27+
},
28+
"22.04": {
29+
"FROM": "docker://ubuntu:22.04"
30+
},
31+
"24.04": {
32+
"FROM": "docker://ubuntu:24.04"
33+
},
34+
"25.04": {
35+
"FROM": "docker://ubuntu:25.04"
36+
}
37+
},
38+
"default_version": "24.04"
39+
},
40+
"rhel": {
41+
"FROM": "docker://registry.access.redhat.com/ubi9",
42+
"package-manager-update-cmd": "dnf update -y",
43+
"package-manager-get-cmd": "dnf install -y",
44+
"packages": [
45+
"python3",
46+
"python-pip",
47+
"git",
48+
"wget",
49+
"sudo"
50+
],
51+
"versions": {
52+
"8": {
53+
"FROM": "docker://registry.access.redhat.com/ubi8",
54+
"packages": [
55+
"python39",
56+
"python39-pip",
57+
"git",
58+
"wget",
59+
"sudo"
60+
],
61+
"PYTHON": "python3.9"
62+
},
63+
"9": {}
64+
},
65+
"default_version": "9"
66+
},
67+
"rocky": {
68+
"FROM": "docker://rockylinux:9",
69+
"package-manager-update-cmd": "dnf update -y",
70+
"package-manager-get-cmd": "dnf install -y",
71+
"packages": [
72+
"python3",
73+
"python3-pip",
74+
"git",
75+
"wget",
76+
"sudo"
77+
],
78+
"versions": {
79+
"8": {
80+
"FROM": "docker://rockylinux:8",
81+
"packages": [
82+
"python39",
83+
"python39-pip",
84+
"git",
85+
"wget",
86+
"sudo"
87+
],
88+
"PYTHON": "python3.9"
89+
},
90+
"9": {
91+
"FROM": "docker://rockylinux:9"
92+
}
93+
},
94+
"default_version": "9"
95+
},
96+
"debian": {
97+
"FROM": "docker://debian:12",
98+
"package-manager-update-cmd": "apt-get update -y",
99+
"package-manager-get-cmd": "apt-get install -y",
100+
"packages": [
101+
"python3",
102+
"python3-pip",
103+
"git",
104+
"sudo",
105+
"wget",
106+
"python3-venv",
107+
"unzip",
108+
"ca-certificates"
109+
],
110+
"versions": {
111+
"11": {
112+
"FROM": "docker://debian:11"
113+
},
114+
"12": {
115+
"FROM": "docker://debian:12"
116+
}
117+
},
118+
"default_version": "12"
119+
},
120+
"sles": {
121+
"FROM": "docker://registry.suse.com/bci/bci-base:15.6",
122+
"package-manager-update-cmd": "zypper refresh",
123+
"package-manager-get-cmd": "zypper install -y",
124+
"packages": [
125+
"python311",
126+
"python311-pip",
127+
"git",
128+
"wget",
129+
"sudo",
130+
"unzip",
131+
"ca-certificates",
132+
"gzip",
133+
"tar",
134+
"gawk"
135+
],
136+
"versions": {
137+
"15.4": {
138+
"FROM": "docker://registry.suse.com/bci/bci-base:15.4",
139+
"packages": [
140+
"python310",
141+
"python310-pip",
142+
"git",
143+
"wget",
144+
"sudo",
145+
"unzip",
146+
"ca-certificates",
147+
"gzip",
148+
"tar",
149+
"gawk"
150+
],
151+
"PYTHON": "python3.10"
152+
},
153+
"15.5": {
154+
"FROM": "docker://registry.suse.com/bci/bci-base:15.5"
155+
},
156+
"15.6": {
157+
"FROM": "docker://registry.suse.com/bci/bci-base:15.6"
158+
}
159+
},
160+
"default_version": "15.6",
161+
"PYTHON": "python3.11"
162+
},
163+
"arch": {
164+
"FROM": "docker://archlinux",
165+
"package-manager-update-cmd": "pacman -Syu --noconfirm",
166+
"package-manager-get-cmd": "pacman -Sy --noconfirm",
167+
"packages": [
168+
"python",
169+
"python-pip",
170+
"git",
171+
"wget",
172+
"sudo"
173+
],
174+
"versions": {
175+
"latest": {}
176+
},
177+
"default_version": "latest"
178+
},
179+
"amazonlinux": {
180+
"FROM": "docker://amazonlinux:2023",
181+
"package-manager-update-cmd": "dnf update -y",
182+
"package-manager-get-cmd": "dnf install -y",
183+
"packages": [
184+
"python3",
185+
"python3-pip",
186+
"git",
187+
"wget",
188+
"sudo",
189+
"tar",
190+
"gzip",
191+
"unzip"
192+
],
193+
"versions": {
194+
"2023": {
195+
"FROM": "docker://amazonlinux:2023"
196+
}
197+
},
198+
"default_version": "2023"
199+
}
200+
}
201+
}

0 commit comments

Comments
 (0)