Skip to content

Commit 764cdb3

Browse files
elezarcdesiniotis
elezar
and
cdesiniotis
committed
Add support for drop-in configs
This change adds explicit support for drop-in configs as supported by containerd and cri-o. Signed-off-by: Evan Lezar <elezar@nvidia.com> Co-authored-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent 39e13ab commit 764cdb3

File tree

2 files changed

+184
-29
lines changed

2 files changed

+184
-29
lines changed

‎controllers/object_controls.go‎

Lines changed: 77 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,18 @@ import (
5757
const (
5858
// DefaultContainerdConfigFile indicates default config file path for containerd
5959
DefaultContainerdConfigFile = "/etc/containerd/config.toml"
60+
// DefaultContainerdDropInConfigFile indicates default drop-in config file path for containerd
61+
DefaultContainerdDropInConfigFile = "/etc/containerd/conf.d/99-nvidia.toml"
6062
// DefaultContainerdSocketFile indicates default containerd socket file
6163
DefaultContainerdSocketFile = "/run/containerd/containerd.sock"
6264
// DefaultDockerConfigFile indicates default config file path for docker
6365
DefaultDockerConfigFile = "/etc/docker/daemon.json"
6466
// DefaultDockerSocketFile indicates default docker socket file
6567
DefaultDockerSocketFile = "/var/run/docker.sock"
66-
// DefaultCRIOConfigFile indicates default config file path for cri-o.
67-
// Note, config files in the drop-in directory, /etc/crio/crio.conf.d,
68-
// have a higher priority than the default /etc/crio/crio.conf file.
69-
DefaultCRIOConfigFile = "/etc/crio/crio.conf.d/99-nvidia.conf"
68+
// DefaultCRIOConfigFile indicates default config file path for cri-o. .
69+
DefaultCRIOConfigFile = "/etc/crio/config.toml"
70+
// DefaultCRIODropInConfigFile indicates the default path to the drop-in config file for cri-o
71+
DefaultCRIODropInConfigFile = "/etc/crio/crio.conf.d/99-nvidia.conf"
7072
// TrustedCAConfigMapName indicates configmap with custom user CA injected
7173
TrustedCAConfigMapName = "gpu-operator-trusted-ca"
7274
// TrustedCABundleFileName indicates custom user ca certificate filename
@@ -95,6 +97,8 @@ const (
9597
DefaultRuntimeSocketTargetDir = "/runtime/sock-dir/"
9698
// DefaultRuntimeConfigTargetDir represents target directory where runtime socket dirctory will be mounted
9799
DefaultRuntimeConfigTargetDir = "/runtime/config-dir/"
100+
// DefaultRuntimeDropInConfigTargetDir represents target directory where drop-in config directory will be mounted
101+
DefaultRuntimeDropInConfigTargetDir = "/runtime/config-dir.d/"
98102
// ValidatorImageEnvName indicates env name for validator image passed
99103
ValidatorImageEnvName = "VALIDATOR_IMAGE"
100104
// ValidatorImagePullPolicyEnvName indicates env name for validator image pull policy passed
@@ -1355,12 +1359,18 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
13551359
setContainerEnv(mainContainer, CRIOConfigModeEnvName, "config")
13561360
}
13571361

1362+
// For runtime config files we have top-level configs and drop-in files.
1363+
// These are supported as follows:
1364+
// * Docker only supports top-level config files.
1365+
// * Containerd supports drop-in files, but required modification to the top-level config
1366+
// * Crio supports drop-in files at a predefined location. The top-level config may be read
1367+
// but should not be updated.
1368+
13581369
// setup mounts for runtime config file
1359-
runtimeConfigFile, err := getRuntimeConfigFile(mainContainer, runtime)
1370+
topLevelConfigFile, dropInConfigFile, err := getRuntimeConfigFiles(mainContainer, runtime)
13601371
if err != nil {
1361-
return fmt.Errorf("error getting path to runtime config file: %v", err)
1372+
return fmt.Errorf("error getting path to runtime config file: %w", err)
13621373
}
1363-
sourceConfigFileName := path.Base(runtimeConfigFile)
13641374

13651375
var configEnvvarName string
13661376
switch runtime {
@@ -1372,15 +1382,43 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
13721382
configEnvvarName = "CRIO_CONFIG"
13731383
}
13741384

1375-
setContainerEnv(mainContainer, "RUNTIME_CONFIG", DefaultRuntimeConfigTargetDir+sourceConfigFileName)
1376-
setContainerEnv(mainContainer, configEnvvarName, DefaultRuntimeConfigTargetDir+sourceConfigFileName)
1385+
// Handle the top-level configs
1386+
if topLevelConfigFile != "" {
1387+
sourceConfigFileName := path.Base(topLevelConfigFile)
1388+
sourceConfigDir := path.Dir(topLevelConfigFile)
1389+
containerConfigDir := DefaultRuntimeConfigTargetDir
1390+
setContainerEnv(mainContainer, "RUNTIME_CONFIG", containerConfigDir+sourceConfigFileName)
1391+
setContainerEnv(mainContainer, configEnvvarName, containerConfigDir+sourceConfigFileName)
1392+
1393+
volMountConfigName := fmt.Sprintf("%s-config", runtime)
1394+
volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: containerConfigDir}
1395+
mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
1396+
1397+
configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: sourceConfigDir, Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
1398+
obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
1399+
}
1400+
1401+
// Handle the drop-in configs
1402+
// TODO: It's a bit of a hack to skip the `nvidia-kata-manager` container here.
1403+
// Ideally if the two projects are using the SAME API then this should be
1404+
// captured more rigorously.
1405+
// Note that we probably want to implement drop-in file support in the
1406+
// kata manager in any case -- in which case it will be good to use a
1407+
// similar implementation.
1408+
if dropInConfigFile != "" && containerName != "nvidia-kata-manager" {
1409+
sourceConfigFileName := path.Base(dropInConfigFile)
1410+
sourceConfigDir := path.Dir(dropInConfigFile)
1411+
containerConfigDir := DefaultRuntimeDropInConfigTargetDir
1412+
setContainerEnv(mainContainer, "RUNTIME_DROP_IN_CONFIG", containerConfigDir+sourceConfigFileName)
1413+
setContainerEnv(mainContainer, "RUNTIME_DROP_IN_CONFIG_HOST_PATH", dropInConfigFile)
13771414

1378-
volMountConfigName := fmt.Sprintf("%s-config", runtime)
1379-
volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: DefaultRuntimeConfigTargetDir}
1380-
mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
1415+
volMountConfigName := fmt.Sprintf("%s-drop-in-config", runtime)
1416+
volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: containerConfigDir}
1417+
mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
13811418

1382-
configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(runtimeConfigFile), Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
1383-
obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
1419+
configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: sourceConfigDir, Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
1420+
obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
1421+
}
13841422

13851423
// setup mounts for runtime socket file
13861424
runtimeSocketFile, err := getRuntimeSocketFile(mainContainer, runtime)
@@ -2396,30 +2434,43 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol
23962434
return nil
23972435
}
23982436

2399-
// get runtime(docker, containerd) config file path based on toolkit container env or default
2400-
func getRuntimeConfigFile(c *corev1.Container, runtime string) (string, error) {
2401-
var runtimeConfigFile string
2437+
// getRuntimeConfigFiles returns the path to the top-level and drop-in config files that
2438+
// should be used when configuring the specified container runtime.
2439+
func getRuntimeConfigFiles(c *corev1.Container, runtime string) (string, string, error) {
24022440
switch runtime {
24032441
case gpuv1.Docker.String():
2404-
runtimeConfigFile = DefaultDockerConfigFile
2442+
topLevelConfigFile := DefaultDockerConfigFile
24052443
if value := getContainerEnv(c, "DOCKER_CONFIG"); value != "" {
2406-
runtimeConfigFile = value
2444+
topLevelConfigFile = value
24072445
}
2446+
// Docker does not support drop-in files.
2447+
return topLevelConfigFile, "", nil
24082448
case gpuv1.Containerd.String():
2409-
runtimeConfigFile = DefaultContainerdConfigFile
2449+
topLevelConfigFile := DefaultContainerdConfigFile
2450+
// TODO: We should also read RUNTIME_CONFIG here
24102451
if value := getContainerEnv(c, "CONTAINERD_CONFIG"); value != "" {
2411-
runtimeConfigFile = value
2452+
topLevelConfigFile = value
2453+
}
2454+
dropInConfigFile := DefaultContainerdDropInConfigFile
2455+
if value := getContainerEnv(c, "RUNTIME_DROP_IN_CONFIG"); value != "" {
2456+
dropInConfigFile = value
24122457
}
2458+
return topLevelConfigFile, dropInConfigFile, nil
24132459
case gpuv1.CRIO.String():
2414-
runtimeConfigFile = DefaultCRIOConfigFile
2460+
// TODO: We should still allow the top-level config to be specified
2461+
// TODO: We should also read RUNTIME_CONFIG here
2462+
topLevelConfigFile := DefaultCRIOConfigFile
24152463
if value := getContainerEnv(c, "CRIO_CONFIG"); value != "" {
2416-
runtimeConfigFile = value
2464+
topLevelConfigFile = value
2465+
}
2466+
dropInConfigFile := DefaultCRIODropInConfigFile
2467+
if value := getContainerEnv(c, "RUNTIME_DROP_IN_CONFIG"); value != "" {
2468+
dropInConfigFile = value
24172469
}
2470+
return topLevelConfigFile, dropInConfigFile, nil
24182471
default:
2419-
return "", fmt.Errorf("invalid runtime: %s", runtime)
2472+
return "", "", fmt.Errorf("invalid runtime: %s", runtime)
24202473
}
2421-
2422-
return runtimeConfigFile, nil
24232474
}
24242475

24252476
// get runtime(docker, containerd) socket file path based on toolkit container env or default

‎controllers/transforms_test.go‎

Lines changed: 107 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@ func TestTransformForRuntime(t *testing.T) {
332332
WithContainer(corev1.Container{Name: "test-ctr"}),
333333
expectedOutput: NewDaemonset().
334334
WithHostPathVolume("containerd-config", filepath.Dir(DefaultContainerdConfigFile), newHostPathType(corev1.HostPathDirectoryOrCreate)).
335+
WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", newHostPathType(corev1.HostPathDirectoryOrCreate)).
335336
WithHostPathVolume("containerd-socket", filepath.Dir(DefaultContainerdSocketFile), nil).
336337
WithContainer(corev1.Container{
337338
Name: "test-ctr",
@@ -340,11 +341,14 @@ func TestTransformForRuntime(t *testing.T) {
340341
{Name: "CONTAINERD_RUNTIME_CLASS", Value: DefaultRuntimeClass},
341342
{Name: "RUNTIME_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultContainerdConfigFile))},
342343
{Name: "CONTAINERD_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultContainerdConfigFile))},
344+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.toml"},
345+
{Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"},
343346
{Name: "RUNTIME_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))},
344347
{Name: "CONTAINERD_SOCKET", Value: filepath.Join(DefaultRuntimeSocketTargetDir, filepath.Base(DefaultContainerdSocketFile))},
345348
},
346349
VolumeMounts: []corev1.VolumeMount{
347350
{Name: "containerd-config", MountPath: DefaultRuntimeConfigTargetDir},
351+
{Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"},
348352
{Name: "containerd-socket", MountPath: DefaultRuntimeSocketTargetDir},
349353
},
350354
}),
@@ -354,17 +358,21 @@ func TestTransformForRuntime(t *testing.T) {
354358
runtime: gpuv1.CRIO,
355359
input: NewDaemonset().WithContainer(corev1.Container{Name: "test-ctr"}),
356360
expectedOutput: NewDaemonset().
357-
WithHostPathVolume("crio-config", filepath.Dir(DefaultCRIOConfigFile), newHostPathType(corev1.HostPathDirectoryOrCreate)).
361+
WithHostPathVolume("crio-config", "/etc/crio", newHostPathType(corev1.HostPathDirectoryOrCreate)).
362+
WithHostPathVolume("crio-drop-in-config", "/etc/crio/crio.conf.d", newHostPathType(corev1.HostPathDirectoryOrCreate)).
358363
WithContainer(corev1.Container{
359364
Name: "test-ctr",
360365
Env: []corev1.EnvVar{
361366
{Name: "RUNTIME", Value: gpuv1.CRIO.String()},
362367
{Name: CRIOConfigModeEnvName, Value: "config"},
363-
{Name: "RUNTIME_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultCRIOConfigFile))},
364-
{Name: "CRIO_CONFIG", Value: filepath.Join(DefaultRuntimeConfigTargetDir, filepath.Base(DefaultCRIOConfigFile))},
368+
{Name: "RUNTIME_CONFIG", Value: "/runtime/config-dir/config.toml"},
369+
{Name: "CRIO_CONFIG", Value: "/runtime/config-dir/config.toml"},
370+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.conf"},
371+
{Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/crio/crio.conf.d/99-nvidia.conf"},
365372
},
366373
VolumeMounts: []corev1.VolumeMount{
367374
{Name: "crio-config", MountPath: DefaultRuntimeConfigTargetDir},
375+
{Name: "crio-drop-in-config", MountPath: "/runtime/config-dir.d/"},
368376
},
369377
}),
370378
},
@@ -657,15 +665,19 @@ func TestTransformToolkit(t *testing.T) {
657665
{Name: "CONTAINERD_RUNTIME_CLASS", Value: "nvidia"},
658666
{Name: "RUNTIME_CONFIG", Value: "/runtime/config-dir/config.toml"},
659667
{Name: "CONTAINERD_CONFIG", Value: "/runtime/config-dir/config.toml"},
668+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.toml"},
669+
{Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"},
660670
{Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"},
661671
{Name: "CONTAINERD_SOCKET", Value: "/runtime/sock-dir/containerd.sock"},
662672
},
663673
VolumeMounts: []corev1.VolumeMount{
664674
{Name: "containerd-config", MountPath: "/runtime/config-dir/"},
675+
{Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"},
665676
{Name: "containerd-socket", MountPath: "/runtime/sock-dir/"},
666677
},
667678
}).
668679
WithHostPathVolume("containerd-config", "/etc/containerd", newHostPathType(corev1.HostPathDirectoryOrCreate)).
680+
WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", newHostPathType(corev1.HostPathDirectoryOrCreate)).
669681
WithHostPathVolume("containerd-socket", "/run/containerd", nil).
670682
WithPullSecret("pull-secret"),
671683
},
@@ -731,14 +743,18 @@ func TestTransformToolkit(t *testing.T) {
731743
{Name: "CONTAINERD_SET_AS_DEFAULT", Value: "true"},
732744
{Name: "RUNTIME", Value: "containerd"},
733745
{Name: "RUNTIME_CONFIG", Value: "/runtime/config-dir/config.toml"},
746+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/runtime/config-dir.d/99-nvidia.toml"},
747+
{Name: "RUNTIME_DROP_IN_CONFIG_HOST_PATH", Value: "/etc/containerd/conf.d/99-nvidia.toml"},
734748
{Name: "RUNTIME_SOCKET", Value: "/runtime/sock-dir/containerd.sock"},
735749
},
736750
VolumeMounts: []corev1.VolumeMount{
737751
{Name: "containerd-config", MountPath: "/runtime/config-dir/"},
752+
{Name: "containerd-drop-in-config", MountPath: "/runtime/config-dir.d/"},
738753
{Name: "containerd-socket", MountPath: "/runtime/sock-dir/"},
739754
},
740755
}).
741756
WithHostPathVolume("containerd-config", "/var/lib/rancher/k3s/agent/etc/containerd", newHostPathType(corev1.HostPathDirectoryOrCreate)).
757+
WithHostPathVolume("containerd-drop-in-config", "/etc/containerd/conf.d", newHostPathType(corev1.HostPathDirectoryOrCreate)).
742758
WithHostPathVolume("containerd-socket", "/run/k3s/containerd", nil).
743759
WithPullSecret("pull-secret"),
744760
},
@@ -2261,3 +2277,91 @@ func TestTransformDevicePluginCtrForCDI(t *testing.T) {
22612277
})
22622278
}
22632279
}
2280+
2281+
func TestGetRuntimeConfigFiles(t *testing.T) {
2282+
testCases := []struct {
2283+
description string
2284+
container corev1.Container
2285+
runtime string
2286+
expectedTopLevelConfigFile string
2287+
expectedDropInConfigFile string
2288+
errorExpected bool
2289+
}{
2290+
{
2291+
description: "invalid runtime",
2292+
container: corev1.Container{},
2293+
runtime: "foo",
2294+
errorExpected: true,
2295+
},
2296+
{
2297+
description: "docker",
2298+
container: corev1.Container{},
2299+
runtime: gpuv1.Docker.String(),
2300+
expectedTopLevelConfigFile: DefaultDockerConfigFile,
2301+
expectedDropInConfigFile: "",
2302+
},
2303+
{
2304+
description: "docker, config path overridden",
2305+
container: corev1.Container{
2306+
Env: []corev1.EnvVar{
2307+
{Name: "DOCKER_CONFIG", Value: "/path/to/docker/daemon.json"},
2308+
},
2309+
},
2310+
runtime: gpuv1.Docker.String(),
2311+
expectedTopLevelConfigFile: "/path/to/docker/daemon.json",
2312+
expectedDropInConfigFile: "",
2313+
},
2314+
{
2315+
description: "containerd",
2316+
container: corev1.Container{},
2317+
runtime: gpuv1.Containerd.String(),
2318+
expectedTopLevelConfigFile: DefaultContainerdConfigFile,
2319+
expectedDropInConfigFile: DefaultContainerdDropInConfigFile,
2320+
},
2321+
{
2322+
description: "containerd, config path overridden",
2323+
container: corev1.Container{
2324+
Env: []corev1.EnvVar{
2325+
{Name: "CONTAINERD_CONFIG", Value: "/path/to/containerd/config.toml"},
2326+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/path/to/containerd/drop-in/config.toml"},
2327+
},
2328+
},
2329+
runtime: gpuv1.Containerd.String(),
2330+
expectedTopLevelConfigFile: "/path/to/containerd/config.toml",
2331+
expectedDropInConfigFile: "/path/to/containerd/drop-in/config.toml",
2332+
},
2333+
{
2334+
description: "crio",
2335+
container: corev1.Container{},
2336+
runtime: gpuv1.CRIO.String(),
2337+
expectedTopLevelConfigFile: DefaultCRIOConfigFile,
2338+
expectedDropInConfigFile: DefaultCRIODropInConfigFile,
2339+
},
2340+
{
2341+
description: "crio, config path overridden",
2342+
container: corev1.Container{
2343+
Env: []corev1.EnvVar{
2344+
{Name: "CRIO_CONFIG", Value: "/path/to/crio/config.toml"},
2345+
{Name: "RUNTIME_DROP_IN_CONFIG", Value: "/path/to/crio/drop-in/config.toml"},
2346+
},
2347+
},
2348+
runtime: gpuv1.CRIO.String(),
2349+
expectedTopLevelConfigFile: "/path/to/crio/config.toml",
2350+
expectedDropInConfigFile: "/path/to/crio/drop-in/config.toml",
2351+
},
2352+
}
2353+
2354+
for _, tc := range testCases {
2355+
t.Run(tc.description, func(t *testing.T) {
2356+
topLevelConfigFile, dropInConfigFile, err := getRuntimeConfigFiles(&tc.container, tc.runtime)
2357+
if tc.errorExpected {
2358+
require.Error(t, err)
2359+
return
2360+
}
2361+
require.NoError(t, err)
2362+
require.EqualValues(t, tc.expectedTopLevelConfigFile, topLevelConfigFile)
2363+
require.EqualValues(t, tc.expectedDropInConfigFile, dropInConfigFile)
2364+
})
2365+
}
2366+
2367+
}

0 commit comments

Comments
 (0)