Skip to content

fix: use inventory-file for node scanner #1530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MaxRink wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: use inventory-file for node scanner #1530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions controllers/nodes/deployment_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"go.mondoo.com/mondoo-operator/api/v1alpha2"
"go.mondoo.com/mondoo-operator/pkg/client/mondooclient"
"go.mondoo.com/mondoo-operator/pkg/constants"
"go.mondoo.com/mondoo-operator/pkg/utils/k8s"
"go.mondoo.com/mondoo-operator/pkg/utils/mondoo"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -75,6 +76,11 @@ func (n *DeploymentHandler) syncCronJob(ctx context.Context) error {
logger.Error(err, "Failed to resolve mondoo-client container image")
return err
}
renderImage, err := n.ContainerImageResolver.ContainerImage(ctx, constants.BusyBoxImage, n.MondooOperatorConfig.Spec.SkipContainerResolution)
if err != nil {
logger.Error(err, "Failed to resolve node inventory render container image")
return err
}

clusterUid, err := k8s.GetClusterUID(ctx, n.KubeClient, logger)
if err != nil {
Expand Down Expand Up @@ -109,7 +115,7 @@ func (n *DeploymentHandler) syncCronJob(ctx context.Context) error {
return err
}

desired := CronJob(mondooClientImage, node, n.Mondoo, n.IsOpenshift, *n.MondooOperatorConfig)
desired := CronJob(mondooClientImage, renderImage, node, n.Mondoo, n.IsOpenshift, *n.MondooOperatorConfig)
cronJob := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}}
op, err := k8s.CreateOrUpdate(ctx, n.KubeClient, cronJob, n.Mondoo, logger, func() error {
k8s.UpdateCronJobFields(cronJob, desired)
Expand Down Expand Up @@ -180,6 +186,11 @@ func (n *DeploymentHandler) syncDaemonSet(ctx context.Context) error {
logger.Error(err, "Failed to resolve mondoo-client container image")
return err
}
renderImage, err := n.ContainerImageResolver.ContainerImage(ctx, constants.BusyBoxImage, n.MondooOperatorConfig.Spec.SkipContainerResolution)
if err != nil {
logger.Error(err, "Failed to resolve node inventory render container image")
return err
}

clusterUid, err := k8s.GetClusterUID(ctx, n.KubeClient, logger)
if err != nil {
Expand Down Expand Up @@ -229,7 +240,7 @@ func (n *DeploymentHandler) syncDaemonSet(ctx context.Context) error {
}
}

desired := DaemonSet(*n.Mondoo, n.IsOpenshift, mondooClientImage, *n.MondooOperatorConfig, slices.Collect(maps.Keys(tolerations)))
desired := DaemonSet(*n.Mondoo, n.IsOpenshift, mondooClientImage, renderImage, *n.MondooOperatorConfig, slices.Collect(maps.Keys(tolerations)))
ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}}
op, err := k8s.CreateOrUpdate(ctx, n.KubeClient, ds, n.Mondoo, logger, func() error {
k8s.UpdateDaemonSetFields(ds, desired)
Expand Down
60 changes: 48 additions & 12 deletions controllers/nodes/deployment_handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"encoding/json"
"encoding/pem"
"fmt"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -242,7 +243,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateCronJobs() {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, n.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))

cjExpected := CronJob(image, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cjExpected := CronJob(image, constants.BusyBoxImage, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
// Make sure the env vars for both are sorted
utils.SortEnvVars(cjExpected.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
utils.SortEnvVars(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
Expand All @@ -254,6 +255,40 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateCronJobs() {
s.Error(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(gcCj), gcCj))
}

func (s *DeploymentHandlerSuite) TestReconcile_CronJobUsesResolvedRenderImage() {
s.seedNodes()
d := s.createDeploymentHandler()
d.MondooOperatorConfig = &v1alpha2.MondooOperatorConfig{Spec: v1alpha2.MondooOperatorConfigSpec{
SkipContainerResolution: true,
}}
d.ContainerImageResolver = &fakeMondoo.ContainerImageResolverMock{
CnspecImageFunc: func(userImage, userTag, userDigest string, skipResolveImage bool) (string, error) {
return "registry.example.com/cnspec:13-rootless", nil
},
ContainerImageFunc: func(ctx context.Context, image string, skipResolveImage bool) (string, error) {
s.Equal(constants.BusyBoxImage, image)
s.True(skipResolveImage)
return "registry.example.com/dockerhub/library/busybox:1.36", nil
},
}

mondooAuditConfig := &s.auditConfig
s.NoError(d.KubeClient.Create(s.ctx, mondooAuditConfig))

result, err := d.Reconcile(s.ctx)
s.NoError(err)
s.True(result.IsZero())

nodes := &corev1.NodeList{}
s.NoError(d.KubeClient.List(s.ctx, nodes))
for _, n := range nodes.Items {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, n.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))
s.Len(cj.Spec.JobTemplate.Spec.Template.Spec.InitContainers, 1)
s.Equal("registry.example.com/dockerhub/library/busybox:1.36", cj.Spec.JobTemplate.Spec.Template.Spec.InitContainers[0].Image)
}
}

func (s *DeploymentHandlerSuite) TestReconcile_CreateCronJobs_CustomEnvVars() {
s.seedNodes()
d := s.createDeploymentHandler()
Expand All @@ -276,7 +311,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateCronJobs_CustomEnvVars() {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, n.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))

cjExpected := CronJob(image, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cjExpected := CronJob(image, constants.BusyBoxImage, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
// Make sure the env vars for both are sorted
utils.SortEnvVars(cjExpected.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
utils.SortEnvVars(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
Expand Down Expand Up @@ -309,7 +344,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateCronJobs_Switch() {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, n.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))

cjExpected := CronJob(image, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cjExpected := CronJob(image, constants.BusyBoxImage, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
// Make sure the env vars for both are sorted
utils.SortEnvVars(cjExpected.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
utils.SortEnvVars(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
Expand Down Expand Up @@ -349,7 +384,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_UpdateCronJobs() {
s.NoError(err)

// Make sure a cron job exists for one of the nodes
cj := CronJob(image, nodes.Items[1], &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cj := CronJob(image, constants.BusyBoxImage, nodes.Items[1], &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{"test-command"}
s.NoError(d.KubeClient.Create(s.ctx, cj))

Expand All @@ -361,7 +396,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_UpdateCronJobs() {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, n.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))

cjExpected := CronJob(image, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cjExpected := CronJob(image, constants.BusyBoxImage, n, &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
// Make sure the env vars for both are sorted
utils.SortEnvVars(cjExpected.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
utils.SortEnvVars(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
Expand Down Expand Up @@ -407,7 +442,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CleanCronJobsForDeletedNodes() {
cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: CronJobName(s.auditConfig.Name, nodes.Items[0].Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(cj), cj))

cjExpected := CronJob(image, nodes.Items[0], &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
cjExpected := CronJob(image, constants.BusyBoxImage, nodes.Items[0], &s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
// Make sure the env vars for both are sorted
utils.SortEnvVars(cjExpected.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
utils.SortEnvVars(cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env)
Expand Down Expand Up @@ -435,7 +470,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateDaemonSets() {
ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: DaemonSetName(s.auditConfig.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(ds), ds))

dsExpected := DaemonSet(s.auditConfig, false, image, v1alpha2.MondooOperatorConfig{},
dsExpected := DaemonSet(s.auditConfig, false, image, constants.BusyBoxImage, v1alpha2.MondooOperatorConfig{},
[]corev1.Toleration{{Key: "node-role.kubernetes.io/master", Value: "true", Effect: corev1.TaintEffectNoExecute}})
// Make sure the env vars for both are sorted
utils.SortEnvVars(dsExpected.Spec.Template.Spec.Containers[0].Env)
Expand Down Expand Up @@ -468,7 +503,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateDaemonSets_Switch() {
ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: DaemonSetName(s.auditConfig.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(ds), ds))

dsExpected := DaemonSet(s.auditConfig, false, image, v1alpha2.MondooOperatorConfig{},
dsExpected := DaemonSet(s.auditConfig, false, image, constants.BusyBoxImage, v1alpha2.MondooOperatorConfig{},
[]corev1.Toleration{{Key: "node-role.kubernetes.io/master", Value: "true", Effect: corev1.TaintEffectNoExecute}})
s.Equal(dsExpected.Spec, ds.Spec)

Expand Down Expand Up @@ -506,7 +541,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_UpdateDaemonSets() {
s.NoError(err)

// Make sure a daemonset exists
ds := DaemonSet(s.auditConfig, false, image, v1alpha2.MondooOperatorConfig{}, nil)
ds := DaemonSet(s.auditConfig, false, image, constants.BusyBoxImage, v1alpha2.MondooOperatorConfig{}, nil)
ds.Spec.Template.Spec.Containers[0].Command = []string{"test-command"}
s.NoError(d.KubeClient.Create(s.ctx, ds))

Expand All @@ -517,7 +552,7 @@ func (s *DeploymentHandlerSuite) TestReconcile_UpdateDaemonSets() {
ds = &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: DaemonSetName(s.auditConfig.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(ds), ds))

depExpected := DaemonSet(s.auditConfig, false, image, v1alpha2.MondooOperatorConfig{},
depExpected := DaemonSet(s.auditConfig, false, image, constants.BusyBoxImage, v1alpha2.MondooOperatorConfig{},
[]corev1.Toleration{{Key: "node-role.kubernetes.io/master", Value: "true", Effect: corev1.TaintEffectNoExecute}})
s.Equal(depExpected.Spec, ds.Spec)
}
Expand Down Expand Up @@ -765,8 +800,9 @@ func (s *DeploymentHandlerSuite) TestReconcile_Deployment_CustomInterval() {
ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: DaemonSetName(s.auditConfig.Name), Namespace: s.auditConfig.Namespace}}
s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(ds), ds))

s.Contains(ds.Spec.Template.Spec.Containers[0].Command, "--timer")
s.Contains(ds.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("%d", s.auditConfig.Spec.Nodes.IntervalTimer))
command := strings.Join(ds.Spec.Template.Spec.Containers[0].Command, " ")
s.Contains(command, "--timer")
s.Contains(command, fmt.Sprintf("%d", s.auditConfig.Spec.Nodes.IntervalTimer))
}

func (s *DeploymentHandlerSuite) TestGarbageCollection_RunsAfterSuccessfulScan() {
Expand Down
79 changes: 72 additions & 7 deletions controllers/nodes/resources.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import (
"crypto/sha256"
"fmt"
"math"
"strings"

appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"sigs.k8s.io/yaml"
Expand All @@ -35,16 +37,19 @@ const (

ignoreQueryAnnotationPrefix = "policies.k8s.mondoo.com/"

ignoreAnnotationValue = "ignore"
ignoreAnnotationValue = "ignore"
nodeInventoryTemplatePath = "/etc/opt/mondoo/inventory_template.yml"
nodeInventoryRenderedPath = "/tmp/mondoo-node-inventory.yml"
nodeInventoryNodeNamePlaceholder = "__MONDOO_NODE_NAME__"
)

// CronJob creates a CronJob for node scanning
func CronJob(image string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOpenshift bool, cfg v1alpha2.MondooOperatorConfig) *batchv1.CronJob {
func CronJob(image, renderImage string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOpenshift bool, cfg v1alpha2.MondooOperatorConfig) *batchv1.CronJob {
ls := NodeScanningLabels(*m)
cmd := []string{
"cnspec", "scan", "local",
"--config", "/etc/opt/mondoo/mondoo.yml",
"--inventory-template", "/etc/opt/mondoo/inventory_template.yml",
"--inventory-file", nodeInventoryRenderedPath,
}

// Add API proxy if configured (respect SkipProxyForCnspec since node scanning uses cnspec)
Expand Down Expand Up @@ -100,6 +105,9 @@ func CronJob(image string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOp
// The node scanning does not use the Kubernetes API at all, therefore the service account token
// should not be mounted at all.
AutomountServiceAccountToken: ptr.To(false),
InitContainers: []corev1.Container{
renderNodeInventoryInitContainer(node.Name, renderImage),
},
Containers: []corev1.Container{
{
Image: image,
Expand Down Expand Up @@ -187,12 +195,12 @@ func CronJob(image string, node corev1.Node, m *v1alpha2.MondooAuditConfig, isOp
}

// DaemonSet creates a DaemonSet for node scanning
func DaemonSet(m v1alpha2.MondooAuditConfig, isOpenshift bool, image string, cfg v1alpha2.MondooOperatorConfig, tolerations []corev1.Toleration) *appsv1.DaemonSet {
func DaemonSet(m v1alpha2.MondooAuditConfig, isOpenshift bool, image, renderImage string, cfg v1alpha2.MondooOperatorConfig, tolerations []corev1.Toleration) *appsv1.DaemonSet {
labels := NodeScanningLabels(m)
cmd := []string{
"cnspec", "serve",
"--config", "/etc/opt/mondoo/mondoo.yml",
"--inventory-template", "/etc/opt/mondoo/inventory_template.yml",
"--inventory-file", nodeInventoryRenderedPath,
"--timer", fmt.Sprintf("%d", m.Spec.Nodes.IntervalTimer),
}
// Add API proxy if configured (respect SkipProxyForCnspec since node scanning uses cnspec)
Expand Down Expand Up @@ -234,6 +242,9 @@ func DaemonSet(m v1alpha2.MondooAuditConfig, isOpenshift bool, image string, cfg
// should not be mounted at all.
AutomountServiceAccountToken: ptr.To(false),
Tolerations: tolerations,
InitContainers: []corev1.Container{
renderNodeInventoryInitContainer("", renderImage),
},
Containers: []corev1.Container{
{
Image: image,
Expand Down Expand Up @@ -378,12 +389,12 @@ func Inventory(integrationMRN, clusterUID string, m v1alpha2.MondooAuditConfig)
Assets: []*inventory.Asset{
{
Id: "host",
Name: `{{ getenv "NODE_NAME" }}`,
Name: nodeInventoryNodeNamePlaceholder,
Connections: []*inventory.Config{
{
Type: "filesystem",
Host: "/mnt/host",
PlatformId: fmt.Sprintf(`{{ printf "//platformid.api.mondoo.app/runtime/k8s/uid/%%s/node/%%s" "%s" (getenv "NODE_NAME")}}`, clusterUID),
PlatformId: fmt.Sprintf("//platformid.api.mondoo.app/runtime/k8s/uid/%s/node/%s", clusterUID, nodeInventoryNodeNamePlaceholder),
},
},
Labels: map[string]string{
Expand Down Expand Up @@ -420,6 +431,60 @@ func Inventory(integrationMRN, clusterUID string, m v1alpha2.MondooAuditConfig)
return string(invBytes), nil
}

func renderNodeInventoryInitContainer(nodeName, image string) corev1.Container {
render := fmt.Sprintf("sed \"s#%s#${NODE_NAME}#g\" %s > %s",
nodeInventoryNodeNamePlaceholder,
shellQuote(nodeInventoryTemplatePath),
shellQuote(nodeInventoryRenderedPath),
)
env := []corev1.EnvVar{{
Name: "NODE_NAME",
Comment thread
mondoo-code-review[bot] marked this conversation as resolved.
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
},
}}
if nodeName != "" {
env[0].ValueFrom = nil
env[0].Value = nodeName
}

return corev1.Container{
Name: "render-node-inventory",
Image: image,
ImagePullPolicy: corev1.PullIfNotPresent,
Command: []string{"/bin/sh", "-ec", render},
Env: env,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("64Mi"),
},
},
SecurityContext: &corev1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
RunAsNonRoot: ptr.To(true),
RunAsUser: ptr.To(int64(65532)),
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "config", ReadOnly: true, MountPath: "/etc/opt/"},
{Name: "temp", MountPath: "/tmp"},
},
}
}

func shellQuote(arg string) string {
if arg == "" {
return "''"
}
return "'" + strings.ReplaceAll(arg, "'", `'\''`) + "'"
}

func NodeScanningLabels(m v1alpha2.MondooAuditConfig) map[string]string {
return map[string]string{
"app": "mondoo",
Expand Down
Loading
Loading