Skip to content

Commit 2a1992d

Browse files
MaxRinkMaxRink
committed
fix: render node inventory in init container
1 parent 01b9e8c commit 2a1992d

7 files changed

Lines changed: 108 additions & 30 deletions

File tree

‎controllers/nodes/resources.go‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
appsv1 "k8s.io/api/apps/v1"
1313
batchv1 "k8s.io/api/batch/v1"
1414
corev1 "k8s.io/api/core/v1"
15+
"k8s.io/apimachinery/pkg/api/resource"
1516
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1617
"k8s.io/utils/ptr"
1718
"sigs.k8s.io/yaml"
@@ -57,7 +58,6 @@ func CronJob(image string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOp
5758
cmd = append(cmd, "--api-proxy", *apiProxy)
5859
}
5960
}
60-
cmd = renderNodeInventoryCommand(cmd)
6161

6262
var proxyEnvVars []corev1.EnvVar
6363
if !cfg.Spec.SkipProxyForCnspec {
@@ -105,6 +105,9 @@ func CronJob(image string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOp
105105
// The node scanning does not use the Kubernetes API at all, therefore the service account token
106106
// should not be mounted at all.
107107
AutomountServiceAccountToken: ptr.To(false),
108+
InitContainers: []corev1.Container{
109+
renderNodeInventoryInitContainer(node.Name),
110+
},
108111
Containers: []corev1.Container{
109112
{
110113
Image: image,
@@ -206,7 +209,6 @@ func DaemonSet(m v1alpha2.MondooAuditConfig, isOpenshift bool, image string, cfg
206209
cmd = append(cmd, "--api-proxy", *apiProxy)
207210
}
208211
}
209-
cmd = renderNodeInventoryCommand(cmd)
210212

211213
var proxyEnvVars []corev1.EnvVar
212214
if !cfg.Spec.SkipProxyForCnspec {
@@ -240,6 +242,9 @@ func DaemonSet(m v1alpha2.MondooAuditConfig, isOpenshift bool, image string, cfg
240242
// should not be mounted at all.
241243
AutomountServiceAccountToken: ptr.To(false),
242244
Tolerations: tolerations,
245+
InitContainers: []corev1.Container{
246+
renderNodeInventoryInitContainer(""),
247+
},
243248
Containers: []corev1.Container{
244249
{
245250
Image: image,
@@ -426,21 +431,51 @@ func Inventory(integrationMRN, clusterUID string, m v1alpha2.MondooAuditConfig)
426431
return string(invBytes), nil
427432
}
428433

429-
func renderNodeInventoryCommand(cnspecArgs []string) []string {
434+
func renderNodeInventoryInitContainer(nodeName string) corev1.Container {
430435
render := fmt.Sprintf("sed \"s#%s#${NODE_NAME}#g\" %s > %s",
431436
nodeInventoryNodeNamePlaceholder,
432437
shellQuote(nodeInventoryTemplatePath),
433438
shellQuote(nodeInventoryRenderedPath),
434439
)
435-
return []string{"/bin/sh", "-ec", render + "\nexec " + shellJoin(cnspecArgs)}
436-
}
440+
env := []corev1.EnvVar{{
441+
Name: "NODE_NAME",
442+
ValueFrom: &corev1.EnvVarSource{
443+
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
444+
},
445+
}}
446+
if nodeName != "" {
447+
env[0].ValueFrom = nil
448+
env[0].Value = nodeName
449+
}
437450

438-
func shellJoin(args []string) string {
439-
quoted := make([]string, 0, len(args))
440-
for _, arg := range args {
441-
quoted = append(quoted, shellQuote(arg))
451+
return corev1.Container{
452+
Name: "render-node-inventory",
453+
Image: constants.BusyBoxImage,
454+
ImagePullPolicy: corev1.PullIfNotPresent,
455+
Command: []string{"/bin/sh", "-ec", render},
456+
Env: env,
457+
Resources: corev1.ResourceRequirements{
458+
Requests: corev1.ResourceList{
459+
corev1.ResourceCPU: resource.MustParse("10m"),
460+
corev1.ResourceMemory: resource.MustParse("16Mi"),
461+
},
462+
Limits: corev1.ResourceList{
463+
corev1.ResourceCPU: resource.MustParse("50m"),
464+
corev1.ResourceMemory: resource.MustParse("64Mi"),
465+
},
466+
},
467+
SecurityContext: &corev1.SecurityContext{
468+
AllowPrivilegeEscalation: ptr.To(false),
469+
ReadOnlyRootFilesystem: ptr.To(true),
470+
RunAsNonRoot: ptr.To(true),
471+
RunAsUser: ptr.To(int64(65532)),
472+
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
473+
},
474+
VolumeMounts: []corev1.VolumeMount{
475+
{Name: "config", ReadOnly: true, MountPath: "/etc/opt/"},
476+
{Name: "temp", MountPath: "/tmp"},
477+
},
442478
}
443-
return strings.Join(quoted, " ")
444479
}
445480

446481
func shellQuote(arg string) string {

‎controllers/nodes/resources_test.go‎

Lines changed: 46 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -295,14 +295,28 @@ func TestCronJob_UsesInventoryFileFlag(t *testing.T) {
295295
mac := testMondooAuditConfig()
296296

297297
cj := CronJob("test123", testNode, mac, false, v1alpha2.MondooOperatorConfig{})
298-
command := strings.Join(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command, " ")
299-
300-
assert.Contains(t, command, "/bin/sh -ec")
301-
assert.Contains(t, command, "sed")
302-
assert.Contains(t, command, nodeInventoryNodeNamePlaceholder)
303-
assert.Contains(t, command, "--inventory-file")
304-
assert.Contains(t, command, nodeInventoryRenderedPath)
305-
assert.NotContains(t, command, "--inventory-template")
298+
mainCommand := strings.Join(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command, " ")
299+
300+
assert.Contains(t, mainCommand, "cnspec scan local")
301+
assert.Contains(t, mainCommand, "--inventory-file")
302+
assert.Contains(t, mainCommand, nodeInventoryRenderedPath)
303+
assert.NotContains(t, mainCommand, "/bin/sh")
304+
assert.NotContains(t, mainCommand, "sed")
305+
assert.NotContains(t, mainCommand, "--inventory-template")
306+
307+
initContainers := cj.Spec.JobTemplate.Spec.Template.Spec.InitContainers
308+
require.Len(t, initContainers, 1)
309+
init := initContainers[0]
310+
assert.Equal(t, "render-node-inventory", init.Name)
311+
assert.Equal(t, constants.BusyBoxImage, init.Image)
312+
initCommand := strings.Join(init.Command, " ")
313+
assert.Contains(t, initCommand, "/bin/sh -ec")
314+
assert.Contains(t, initCommand, "sed")
315+
assert.Contains(t, initCommand, nodeInventoryNodeNamePlaceholder)
316+
assert.Contains(t, initCommand, nodeInventoryTemplatePath)
317+
assert.Contains(t, initCommand, nodeInventoryRenderedPath)
318+
envMap := envToMap(init.Env)
319+
assert.Equal(t, "test-node-name", envMap["NODE_NAME"])
306320
}
307321

308322
func TestCronJob_SkipProxyForCnspec(t *testing.T) {
@@ -368,14 +382,30 @@ func TestDaemonSet_UsesInventoryFileFlag(t *testing.T) {
368382
mac := *testMondooAuditConfig()
369383

370384
ds := DaemonSet(mac, false, "test123", v1alpha2.MondooOperatorConfig{}, nil)
371-
command := strings.Join(ds.Spec.Template.Spec.Containers[0].Command, " ")
372-
373-
assert.Contains(t, command, "/bin/sh -ec")
374-
assert.Contains(t, command, "sed")
375-
assert.Contains(t, command, nodeInventoryNodeNamePlaceholder)
376-
assert.Contains(t, command, "--inventory-file")
377-
assert.Contains(t, command, nodeInventoryRenderedPath)
378-
assert.NotContains(t, command, "--inventory-template")
385+
mainCommand := strings.Join(ds.Spec.Template.Spec.Containers[0].Command, " ")
386+
387+
assert.Contains(t, mainCommand, "cnspec serve")
388+
assert.Contains(t, mainCommand, "--inventory-file")
389+
assert.Contains(t, mainCommand, nodeInventoryRenderedPath)
390+
assert.NotContains(t, mainCommand, "/bin/sh")
391+
assert.NotContains(t, mainCommand, "sed")
392+
assert.NotContains(t, mainCommand, "--inventory-template")
393+
394+
initContainers := ds.Spec.Template.Spec.InitContainers
395+
require.Len(t, initContainers, 1)
396+
init := initContainers[0]
397+
assert.Equal(t, "render-node-inventory", init.Name)
398+
assert.Equal(t, constants.BusyBoxImage, init.Image)
399+
initCommand := strings.Join(init.Command, " ")
400+
assert.Contains(t, initCommand, "/bin/sh -ec")
401+
assert.Contains(t, initCommand, "sed")
402+
assert.Contains(t, initCommand, nodeInventoryNodeNamePlaceholder)
403+
assert.Contains(t, initCommand, nodeInventoryTemplatePath)
404+
assert.Contains(t, initCommand, nodeInventoryRenderedPath)
405+
require.Len(t, init.Env, 1)
406+
require.NotNil(t, init.Env[0].ValueFrom)
407+
require.NotNil(t, init.Env[0].ValueFrom.FieldRef)
408+
assert.Equal(t, "spec.nodeName", init.Env[0].ValueFrom.FieldRef.FieldPath)
379409
}
380410

381411
func TestDaemonSet_SkipProxyForCnspec(t *testing.T) {

‎docs/user-manual.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,6 +1181,8 @@ spec:
11811181
enable: true
11821182
```
11831183

1184+
Node scanning pods include a lightweight `render-node-inventory` init container that renders the node-specific inventory file before `cnspec` starts. This keeps custom scanner images simple: the scanner image must provide `cnspec`, but it does not need a POSIX shell or `sed`.
1185+
11841186
### Why are (some of) my nodes unscored?
11851187

11861188
In some cases a node scan can require more memory than initially allotted. You can check whether that is the case by running:

‎pkg/constants/images.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,7 @@ const (
1919
// SPIFFE Helper image
2020
// https://github.com/spiffe/spiffe-helper/releases
2121
SPIFFEHelperImage = "ghcr.io/spiffe/spiffe-helper:0.8.0"
22+
23+
// BusyBox image used for lightweight init helpers.
24+
BusyBoxImage = "busybox:1.36"
2225
)

‎pkg/utils/k8s/registry_wif.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ echo "Docker config generated for ACR: ${ACR_LOGIN_SERVER}"
142142
}
143143

144144
default:
145-
image = "busybox:1.36"
145+
image = constants.BusyBoxImage
146146
shell = "/bin/sh"
147147
script = `echo "ERROR: Unknown workload identity provider"; exit 1`
148148
env = []corev1.EnvVar{}

‎pkg/utils/k8s/update_fields.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ func UpdateDeploymentFields(obj, desired *appsv1.Deployment) {
5353
ps := &obj.Spec.Template.Spec
5454
dps := &desired.Spec.Template.Spec
5555
ps.ServiceAccountName = dps.ServiceAccountName
56+
ps.InitContainers = dps.InitContainers
5657
ps.Containers = dps.Containers
5758
ps.Volumes = dps.Volumes
5859
ps.ImagePullSecrets = dps.ImagePullSecrets
@@ -73,6 +74,7 @@ func UpdateDaemonSetFields(obj, desired *appsv1.DaemonSet) {
7374
ps.PriorityClassName = dps.PriorityClassName
7475
ps.AutomountServiceAccountToken = dps.AutomountServiceAccountToken
7576
ps.Tolerations = dps.Tolerations
77+
ps.InitContainers = dps.InitContainers
7678
ps.Containers = dps.Containers
7779
ps.Volumes = dps.Volumes
7880
ps.ImagePullSecrets = dps.ImagePullSecrets

‎pkg/utils/k8s/update_fields_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ func TestUpdateCronJobFields_ImagePullSecrets(t *testing.T) {
2121
Spec: batchv1.JobSpec{
2222
Template: corev1.PodTemplateSpec{
2323
Spec: corev1.PodSpec{
24-
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
24+
InitContainers: []corev1.Container{{Name: "init", Image: "busybox:1.36"}},
25+
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
2526
ImagePullSecrets: []corev1.LocalObjectReference{
2627
{Name: "my-secret"},
2728
{Name: "another-secret"},
@@ -37,6 +38,7 @@ func TestUpdateCronJobFields_ImagePullSecrets(t *testing.T) {
3738
UpdateCronJobFields(obj, desired)
3839

3940
assert.Equal(t, desired.Spec.Schedule, obj.Spec.Schedule)
41+
assert.Equal(t, desired.Spec.JobTemplate.Spec.Template.Spec.InitContainers, obj.Spec.JobTemplate.Spec.Template.Spec.InitContainers)
4042
assert.Equal(t, desired.Spec.JobTemplate.Spec.Template.Spec.Containers, obj.Spec.JobTemplate.Spec.Template.Spec.Containers)
4143
assert.Equal(t, desired.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets, obj.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets)
4244
}
@@ -87,7 +89,8 @@ func TestUpdateDeploymentFields_ImagePullSecrets(t *testing.T) {
8789
Spec: appsv1.DeploymentSpec{
8890
Template: corev1.PodTemplateSpec{
8991
Spec: corev1.PodSpec{
90-
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
92+
InitContainers: []corev1.Container{{Name: "init", Image: "busybox:1.36"}},
93+
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
9194
ImagePullSecrets: []corev1.LocalObjectReference{
9295
{Name: "my-secret"},
9396
},
@@ -100,6 +103,7 @@ func TestUpdateDeploymentFields_ImagePullSecrets(t *testing.T) {
100103
UpdateDeploymentFields(obj, desired)
101104

102105
assert.Equal(t, desired.Spec.Template.Spec.ImagePullSecrets, obj.Spec.Template.Spec.ImagePullSecrets)
106+
assert.Equal(t, desired.Spec.Template.Spec.InitContainers, obj.Spec.Template.Spec.InitContainers)
103107
assert.Equal(t, desired.Spec.Template.Spec.Containers, obj.Spec.Template.Spec.Containers)
104108
}
105109

@@ -108,7 +112,8 @@ func TestUpdateDaemonSetFields_ImagePullSecrets(t *testing.T) {
108112
Spec: appsv1.DaemonSetSpec{
109113
Template: corev1.PodTemplateSpec{
110114
Spec: corev1.PodSpec{
111-
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
115+
InitContainers: []corev1.Container{{Name: "init", Image: "busybox:1.36"}},
116+
Containers: []corev1.Container{{Name: "test", Image: "test:latest"}},
112117
ImagePullSecrets: []corev1.LocalObjectReference{
113118
{Name: "my-secret"},
114119
},
@@ -121,5 +126,6 @@ func TestUpdateDaemonSetFields_ImagePullSecrets(t *testing.T) {
121126
UpdateDaemonSetFields(obj, desired)
122127

123128
assert.Equal(t, desired.Spec.Template.Spec.ImagePullSecrets, obj.Spec.Template.Spec.ImagePullSecrets)
129+
assert.Equal(t, desired.Spec.Template.Spec.InitContainers, obj.Spec.Template.Spec.InitContainers)
124130
assert.Equal(t, desired.Spec.Template.Spec.Containers, obj.Spec.Template.Spec.Containers)
125131
}

0 commit comments

Comments
 (0)