Skip to content

Commit 7bbf5ba

Browse files
committed
Implement an importer plugin mechanism to fetch squashfs files
Signed-off-by: Felix Abecassis <fabecassis@nvidia.com>
1 parent 2dacd4f commit 7bbf5ba

File tree

10 files changed

+363
-60
lines changed

10 files changed

+363
-60
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ CPPFLAGS := -D_GNU_SOURCE -D_FORTIFY_SOURCE=2 -DPYXIS_VERSION=\"$(PYXIS_VER)\" $
1717
CFLAGS := -std=gnu11 -O2 -g -Wall -Wunused-variable -fstack-protector-strong -fpic $(CFLAGS)
1818
LDFLAGS := -Wl,-znoexecstack -Wl,-zrelro -Wl,-znow $(LDFLAGS)
1919

20-
C_SRCS := common.c args.c pyxis_slurmstepd.c pyxis_slurmd.c pyxis_srun.c pyxis_alloc.c pyxis_dispatch.c config.c enroot.c
20+
C_SRCS := common.c args.c pyxis_slurmstepd.c pyxis_slurmd.c pyxis_srun.c pyxis_alloc.c pyxis_dispatch.c config.c enroot.c importer.c
2121
C_OBJS := $(C_SRCS:.c=.o)
2222

2323
DEPS := $(C_OBJS:%.o=%.d)

common.c

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
#include <stdio.h>
77
#include <stdlib.h>
88
#include <string.h>
9+
#include <errno.h>
10+
#include <unistd.h>
11+
12+
#include <slurm/spank.h>
913

1014
#include "common.h"
1115

@@ -128,3 +132,37 @@ void array_free(char ***array, size_t *len)
128132
*array = NULL;
129133
*len = 0;
130134
}
135+
136+
void memfd_print_log(int *log_fd, bool error, const char *tag)
137+
{
138+
int ret;
139+
FILE *fp;
140+
char *line;
141+
142+
ret = lseek(*log_fd, 0, SEEK_SET);
143+
if (ret < 0) {
144+
slurm_info("pyxis: couldn't rewind log file: %s", strerror(errno));
145+
return;
146+
}
147+
148+
fp = fdopen(*log_fd, "r");
149+
if (fp == NULL) {
150+
slurm_info("pyxis: couldn't open in-memory log for printing: %s", strerror(errno));
151+
return;
152+
}
153+
154+
*log_fd = -1;
155+
156+
if (error)
157+
slurm_error("pyxis: printing %s log file:", tag);
158+
while ((line = get_line_from_file(fp)) != NULL) {
159+
if (error)
160+
slurm_error("pyxis: %s", line);
161+
else
162+
slurm_spank_log("%s", line);
163+
free(line);
164+
}
165+
166+
fclose(fp);
167+
return;
168+
}

common.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,6 @@ int array_add_unique(char ***array, size_t *len, const char *entry);
7474

7575
void array_free(char ***array, size_t *len);
7676

77+
void memfd_print_log(int *log_fd, bool error, const char *tag);
78+
7779
#endif /* COMMON_H_ */

config.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ int pyxis_config_parse(struct plugin_config *config, int ac, char **av)
3737
config->container_scope = SCOPE_GLOBAL;
3838
config->sbatch_support = true;
3939
config->use_enroot_load = false;
40+
config->importer_path[0] = '\0';
4041

4142
for (int i = 0; i < ac; ++i) {
4243
if (strncmp("runtime_path=", av[i], 13) == 0) {
@@ -80,6 +81,13 @@ int pyxis_config_parse(struct plugin_config *config, int ac, char **av)
8081
return (-1);
8182
}
8283
config->use_enroot_load = ret;
84+
} else if (strncmp("importer=", av[i], 9) == 0) {
85+
optarg = av[i] + 9;
86+
ret = snprintf(config->importer_path, sizeof(config->importer_path), "%s", optarg);
87+
if (ret < 0 || ret >= sizeof(config->importer_path)) {
88+
slurm_error("pyxis: importer: path too long: %s", optarg);
89+
return (-1);
90+
}
8391
} else {
8492
slurm_error("pyxis: unknown configuration option: %s", av[i]);
8593
return (-1);

config.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
2+
* Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved.
33
*/
44

55
#ifndef CONFIG_H_
@@ -19,6 +19,7 @@ struct plugin_config {
1919
enum container_scope container_scope;
2020
bool sbatch_support;
2121
bool use_enroot_load;
22+
char importer_path[PATH_MAX];
2223
};
2324

2425
int pyxis_config_parse(struct plugin_config *config, int ac, char **av);

enroot.c

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -136,37 +136,6 @@ int enroot_exec_wait(uid_t uid, gid_t gid, int log_fd,
136136
return (0);
137137
}
138138

139-
void enroot_print_log(int log_fd, bool error)
140-
{
141-
int ret;
142-
FILE *fp;
143-
char *line;
144-
145-
ret = lseek(log_fd, 0, SEEK_SET);
146-
if (ret < 0) {
147-
slurm_info("pyxis: couldn't rewind log file: %s", strerror(errno));
148-
return;
149-
}
150-
151-
fp = fdopen(log_fd, "r");
152-
if (fp == NULL) {
153-
slurm_info("pyxis: couldn't open in-memory log for printing: %s", strerror(errno));
154-
return;
155-
}
156-
157-
if (error)
158-
slurm_error("pyxis: printing enroot log file:");
159-
while ((line = get_line_from_file(fp)) != NULL) {
160-
if (error)
161-
slurm_error("pyxis: %s", line);
162-
else
163-
slurm_spank_log("%s", line);
164-
free(line);
165-
}
166-
167-
return;
168-
}
169-
170139
FILE *enroot_exec_output(uid_t uid, gid_t gid,
171140
child_cb callback, char *const argv[])
172141
{
@@ -183,7 +152,10 @@ FILE *enroot_exec_output(uid_t uid, gid_t gid,
183152
ret = enroot_exec_wait(uid, gid, log_fd, callback, argv);
184153
if (ret < 0) {
185154
slurm_error("pyxis: couldn't execute enroot command");
186-
enroot_print_log(log_fd, true);
155+
memfd_print_log(&log_fd, true, "enroot");
156+
if (log_fd >= 0)
157+
close(log_fd);
158+
187159
goto fail;
188160
}
189161

enroot.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,4 @@ int enroot_exec_wait(uid_t uid, gid_t gid, int log_fd,
2020
FILE *enroot_exec_output(uid_t uid, gid_t gid,
2121
child_cb callback, char *const argv[]);
2222

23-
void enroot_print_log(int log_fd, bool error);
24-
2523
#endif /* ENROOT_H_ */

importer.c

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
/*
2+
* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
*/
4+
5+
#include <sys/types.h>
6+
#include <sys/wait.h>
7+
8+
#include <errno.h>
9+
#include <fcntl.h>
10+
#include <stdio.h>
11+
#include <stdlib.h>
12+
#include <string.h>
13+
#include <unistd.h>
14+
15+
#include <slurm/spank.h>
16+
17+
#include "importer.h"
18+
#include "common.h"
19+
20+
static int importer_child_wait(pid_t pid, int *log_fd, const char *cmd)
21+
{
22+
int status;
23+
int ret;
24+
25+
do {
26+
ret = waitpid(pid, &status, 0);
27+
} while (ret < 0 && errno == EINTR);
28+
29+
if (ret < 0) {
30+
slurm_error("pyxis: could not wait for importer %s: %s", cmd, strerror(errno));
31+
return (-1);
32+
}
33+
34+
if (WIFSIGNALED(status)) {
35+
slurm_error("pyxis: importer %s terminated with signal %d", cmd, WTERMSIG(status));
36+
memfd_print_log(log_fd, true, "importer");
37+
return (-1);
38+
}
39+
40+
if (WIFEXITED(status) && WEXITSTATUS(status) != 0) {
41+
slurm_error("pyxis: importer %s failed with exit code %d", cmd, WEXITSTATUS(status));
42+
memfd_print_log(log_fd, true, "importer");
43+
return (-1);
44+
}
45+
46+
return (0);
47+
}
48+
49+
static pid_t importer_exec(const char *importer_path, uid_t uid, gid_t gid,
50+
int stdout_fd, int stderr_fd, child_cb callback, char *const argv[])
51+
{
52+
int ret;
53+
int null_fd = -1;
54+
int oom_score_fd = -1;
55+
pid_t pid;
56+
char *argv_str;
57+
58+
argv_str = join_strings(argv, " ");
59+
if (argv_str != NULL) {
60+
slurm_verbose("pyxis: running importer command: %s", argv_str);
61+
free(argv_str);
62+
}
63+
64+
pid = fork();
65+
if (pid < 0) {
66+
slurm_error("pyxis: fork error: %s", strerror(errno));
67+
return (-1);
68+
}
69+
70+
if (pid == 0) {
71+
null_fd = open("/dev/null", O_RDONLY);
72+
if (null_fd < 0)
73+
_exit(EXIT_FAILURE);
74+
75+
ret = dup2(null_fd, STDIN_FILENO);
76+
if (ret < 0)
77+
_exit(EXIT_FAILURE);
78+
79+
ret = dup2(stdout_fd, STDOUT_FILENO);
80+
if (ret < 0)
81+
_exit(EXIT_FAILURE);
82+
83+
ret = dup2(stderr_fd, STDERR_FILENO);
84+
if (ret < 0)
85+
_exit(EXIT_FAILURE);
86+
87+
oom_score_fd = open("/proc/self/oom_score_adj", O_CLOEXEC | O_WRONLY | O_APPEND);
88+
if (oom_score_fd >= 0) {
89+
dprintf(oom_score_fd, "%d", 0);
90+
close(oom_score_fd);
91+
}
92+
93+
ret = setregid(gid, gid);
94+
if (ret < 0)
95+
_exit(EXIT_FAILURE);
96+
97+
ret = setreuid(uid, uid);
98+
if (ret < 0)
99+
_exit(EXIT_FAILURE);
100+
101+
if (callback != NULL) {
102+
ret = callback();
103+
if (ret < 0)
104+
_exit(EXIT_FAILURE);
105+
}
106+
107+
execve(importer_path, argv, environ);
108+
_exit(EXIT_FAILURE);
109+
}
110+
111+
return (pid);
112+
}
113+
114+
115+
int importer_exec_get(const char *importer_path, uid_t uid, gid_t gid,
116+
child_cb callback, const char *image_uri, char **squashfs_path)
117+
{
118+
char *argv[4];
119+
int log_fd = -1;
120+
int pipe_fds[2] = {-1, -1};
121+
pid_t child;
122+
FILE *pipe_file = NULL;
123+
char *line = NULL;
124+
125+
log_fd = pyxis_memfd_create("importer-log", MFD_CLOEXEC);
126+
if (log_fd < 0) {
127+
slurm_error("pyxis: couldn't create in-memory log file: %s", strerror(errno));
128+
return (-1);
129+
}
130+
131+
/* Create pipe for reading stdout */
132+
if (pipe(pipe_fds) < 0) {
133+
slurm_error("pyxis: could not create pipe: %s", strerror(errno));
134+
xclose(log_fd);
135+
return (-1);
136+
}
137+
138+
argv[0] = (char *)importer_path;
139+
argv[1] = "get";
140+
argv[2] = (char *)image_uri;
141+
argv[3] = NULL;
142+
143+
child = importer_exec(importer_path, uid, gid, pipe_fds[1], log_fd, callback, argv);
144+
xclose(pipe_fds[1]); /* Close write end in parent */
145+
146+
if (child < 0) {
147+
xclose(pipe_fds[0]);
148+
xclose(log_fd);
149+
return (-1);
150+
}
151+
152+
/* Convert pipe fd to FILE* for get_line_from_file */
153+
pipe_file = fdopen(pipe_fds[0], "r");
154+
if (pipe_file == NULL) {
155+
slurm_error("pyxis: could not fdopen pipe: %s", strerror(errno));
156+
xclose(pipe_fds[0]);
157+
xclose(log_fd);
158+
return (-1);
159+
}
160+
161+
/* Read the squashfs path from stdout */
162+
line = get_line_from_file(pipe_file);
163+
fclose(pipe_file); /* This also closes pipe_fds[0] */
164+
165+
/* Wait for child to complete */
166+
if (importer_child_wait(child, &log_fd, "get") < 0) {
167+
free(line);
168+
xclose(log_fd);
169+
return (-1);
170+
}
171+
172+
/* Validate we got a path */
173+
if (line == NULL || strlen(line) == 0) {
174+
slurm_error("pyxis: importer did not return a squashfs path");
175+
memfd_print_log(&log_fd, true, "importer");
176+
free(line);
177+
xclose(log_fd);
178+
return (-1);
179+
}
180+
181+
/* Return the path */
182+
*squashfs_path = line;
183+
184+
slurm_verbose("pyxis: importer squashfs path: %s", *squashfs_path);
185+
186+
xclose(log_fd);
187+
188+
return (0);
189+
}
190+
191+
int importer_exec_release(const char *importer_path, uid_t uid, gid_t gid,
192+
child_cb callback)
193+
{
194+
int ret;
195+
int log_fd;
196+
char *argv[3];
197+
pid_t child;
198+
199+
log_fd = pyxis_memfd_create("importer-log", MFD_CLOEXEC);
200+
if (log_fd < 0) {
201+
slurm_error("pyxis: couldn't create in-memory log file: %s", strerror(errno));
202+
return (-1);
203+
}
204+
205+
argv[0] = (char *)importer_path;
206+
argv[1] = "release";
207+
argv[2] = NULL;
208+
209+
child = importer_exec(importer_path, uid, gid, log_fd, log_fd, callback, argv);
210+
if (child < 0) {
211+
xclose(log_fd);
212+
return (-1);
213+
}
214+
215+
ret = importer_child_wait(child, &log_fd, "release");
216+
if (ret < 0) {
217+
xclose(log_fd);
218+
return (-1);
219+
}
220+
221+
xclose(log_fd);
222+
223+
return (0);
224+
}

0 commit comments

Comments
 (0)