Merge pull request #58 from prachidamle/custom_benchmarks

Custom benchmark for CIS scans

Author: prachidamle

pkg/securityscan/core/configmap.go

@@ -9,6 +9,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8Yaml "k8s.io/apimachinery/pkg/util/yaml"
 
+	wcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/name"
 
 	cisoperatorapiv1 "github.com/rancher/cis-operator/pkg/apis/cis.cattle.io/v1"
@@ -18,12 +19,14 @@ type OverrideSkipInfoData struct {
 	Skip map[string][]string `json:"skip"`
 }
 
-var (
-	CurrentBenchmarkKey = "current"
-	ConfigFileName      = "config.json"
+const (
+	CurrentBenchmarkKey    = "current"
+	ConfigFileName         = "config.json"
+	customBenchmarkBaseDir = "/etc/kbs/custombenchmark/cfg"
 )
 
-func NewConfigMaps(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanprofile *cisoperatorapiv1.ClusterScanProfile, controllerName string, imageConfig *cisoperatorapiv1.ScanImageConfig) (configmaps []*corev1.ConfigMap, err error) {
+func NewConfigMaps(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanprofile *cisoperatorapiv1.ClusterScanProfile, clusterscanbenchmark *cisoperatorapiv1.ClusterScanBenchmark, controllerName string, imageConfig *cisoperatorapiv1.ScanImageConfig, configmapsClient wcorev1.ConfigMapController) (cmMap map[string]*corev1.ConfigMap, err error) {
+	cmMap = make(map[string]*corev1.ConfigMap)
 
 	configdata := map[string]interface{}{
 		"namespace":        cisoperatorapiv1.ClusterScanNS,
@@ -36,35 +39,53 @@ func NewConfigMaps(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanprofile
 	}
 	configcm, err := generateConfigMap(clusterscan, "cisscanConfig.template", "./pkg/securityscan/core/templates/cisscanConfig.template", configdata)
 	if err != nil {
-		return configmaps, err
+		return cmMap, err
 	}
+	cmMap["configcm"] = configcm
+
+	var isCustomBenchmark bool
+	customBenchmarkConfigMapData := make(map[string]string)
+	if clusterscanbenchmark.Spec.CustomBenchmarkConfigMapName != "" {
+		isCustomBenchmark = true
+		customcm, err := getCustomBenchmarkConfigMap(clusterscanbenchmark, configmapsClient)
+		if err != nil {
+			return cmMap, err
+		}
+		customBenchmarkConfigMapData = customcm.Data
+	}
+
 	plugindata := map[string]interface{}{
-		"namespace":         cisoperatorapiv1.ClusterScanNS,
-		"name":              name.SafeConcatName(cisoperatorapiv1.ClusterScanPluginsConfigMap, clusterscan.Name),
-		"runName":           name.SafeConcatName("security-scan-runner", clusterscan.Name),
-		"appName":           "rancher-cis-benchmark",
-		"serviceaccount":    cisoperatorapiv1.ClusterScanSA,
-		"securityScanImage": imageConfig.SecurityScanImage + ":" + imageConfig.SecurityScanImageTag,
-		"benchmarkVersion":  clusterscanprofile.Spec.BenchmarkVersion,
+		"namespace":                    cisoperatorapiv1.ClusterScanNS,
+		"name":                         name.SafeConcatName(cisoperatorapiv1.ClusterScanPluginsConfigMap, clusterscan.Name),
+		"runName":                      name.SafeConcatName("security-scan-runner", clusterscan.Name),
+		"appName":                      "rancher-cis-benchmark",
+		"serviceaccount":               cisoperatorapiv1.ClusterScanSA,
+		"securityScanImage":            imageConfig.SecurityScanImage + ":" + imageConfig.SecurityScanImageTag,
+		"benchmarkVersion":             clusterscanprofile.Spec.BenchmarkVersion,
+		"isCustomBenchmark":            isCustomBenchmark,
+		"configDir":                    customBenchmarkBaseDir,
+		"customBenchmarkConfigMapName": clusterscanbenchmark.Spec.CustomBenchmarkConfigMapName,
+		"customBenchmarkConfigMapData": customBenchmarkConfigMapData,
 	}
 	plugincm, err := generateConfigMap(clusterscan, "pluginConfig.template", "./pkg/securityscan/core/templates/pluginConfig.template", plugindata)
 	if err != nil {
-		return configmaps, err
+		return cmMap, err
 	}
+	cmMap["plugincm"] = plugincm
 
 	var skipConfigcm *corev1.ConfigMap
 	if clusterscanprofile.Spec.SkipTests != nil && len(clusterscanprofile.Spec.SkipTests) > 0 {
 		//create user skip config map as well
 		// create the cm
 		skipDataBytes, err := getOverrideSkipInfoData(clusterscanprofile.Spec.SkipTests)
 		if err != nil {
-			return configmaps, err
+			return cmMap, err
 		}
 		skipConfigcm = getConfigMapObject(getOverrideConfigMapName(clusterscan), string(skipDataBytes))
+		cmMap["skipConfigcm"] = skipConfigcm
 	}
 
-	configmaps = append(configmaps, configcm, plugincm, skipConfigcm)
-	return configmaps, nil
+	return cmMap, nil
 }
 
 func generateConfigMap(clusterscan *cisoperatorapiv1.ClusterScan, templateName string, templateFile string, data map[string]interface{}) (*corev1.ConfigMap, error) {
@@ -120,3 +141,10 @@ func getConfigMapObject(cmName, data string) *corev1.ConfigMap {
 		},
 	}
 }
+
+func getCustomBenchmarkConfigMap(benchmark *cisoperatorapiv1.ClusterScanBenchmark, configmapsClient wcorev1.ConfigMapController) (*corev1.ConfigMap, error) {
+	if benchmark.Spec.CustomBenchmarkConfigMapName == "" {
+		return nil, nil
+	}
+	return configmapsClient.Get(benchmark.Spec.CustomBenchmarkConfigMapNameSpace, benchmark.Spec.CustomBenchmarkConfigMapName, metav1.GetOptions{})
+}

pkg/securityscan/core/templates/pluginConfig.template

@@ -14,7 +14,7 @@ data:
       hostIPC: true
       hostNetwork: true
       hostPID: true
-      serviceAccountName: {{.serviceaccount}}
+      serviceAccountName: {{ .serviceaccount }}
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/controlplane
@@ -38,18 +38,34 @@ data:
       - hostPath:
           path: /etc/cni/net.d
         name: rke2-cni
+      {{- if .isCustomBenchmark }}
+      - configMap:
+          defaultMode: 420
+          items:
+          {{- range $key, $value := .customBenchmarkConfigMapData }}
+          {{- if eq $key "config.yaml"}}
+          - key: {{ $key }}
+            path: {{ $key }}
+          {{- else}}
+          - key: {{ $key }}
+            path: {{ $.benchmarkVersion }}/{{ $key }}
+          {{- end }}
+          {{- end }}
+          name: {{ .customBenchmarkConfigMapName }}
+        name: custom-benchmark-volume
+      {{- end }}
     sonobuoy-config:
       driver: DaemonSet
       plugin-name: rancher-kube-bench
       result-type: rancher-kube-bench
       result-format: raw
     spec:
       name: rancher-kube-bench
-      image: {{.securityScanImage}}
+      image: {{ .securityScanImage }}
       command: ["/bin/bash", "-c", "run_sonobuoy_plugin.sh && sleep 3600"]
       env:
       - name: SONOBUOY_NS
-        value: {{.namespace}}
+        value: {{ .namespace }}
       - name: NODE_NAME
         valueFrom:
           fieldRef:
@@ -59,7 +75,11 @@ data:
       - name: CHROOT_DIR
         value: /node
       - name: OVERRIDE_BENCHMARK_VERSION
-        value: {{.benchmarkVersion}}
+        value: {{ .benchmarkVersion }}
+      {{- if .isCustomBenchmark }}
+      - name: CONFIG_DIR
+        value: {{ .configDir }}
+      {{- end }}
       imagePullPolicy: Always
       securityContext:
         privileged: true
@@ -82,3 +102,7 @@ data:
       - mountPath: /etc/cni/net.d
         name: rke2-cni
         readOnly: true
+      {{- if .isCustomBenchmark }}
+      - mountPath: /etc/kbs/custombenchmark/cfg
+        name: custom-benchmark-volume
+      {{- end }}

pkg/securityscan/jobHandler.go

@@ -60,9 +60,11 @@ func (c *Controller) handleJobs(ctx context.Context) error {
 		// if the scan has completed then delete the job
 		if v1.ClusterScanConditionComplete.IsTrue(scan) {
 			c.ensureCleanup(scan)
-			v1.ClusterScanConditionAlerted.Unknown(scan)
+			if !v1.ClusterScanConditionFailed.IsTrue(scan) {
+				logrus.Infof("Marking ClusterScanConditionAlerted for scan: %v", scanName)
+				v1.ClusterScanConditionAlerted.Unknown(scan)
+			}
 			scan.Status.ObservedGeneration = scan.Generation
-			logrus.Infof("Marking ClusterScanConditionAlerted for scan: %v", scanName)
 			c.setClusterScanStatusDisplay(scan)
 
 			if scan.Spec.ScheduledScanConfig != nil && scan.Spec.ScheduledScanConfig.CronSchedule != "" {

pkg/securityscan/scanHandler.go

@@ -99,22 +99,25 @@ func (c *Controller) handleClusterScans(ctx context.Context) error {
 				}
 				//launch new on demand scan
 				c.mu.Lock()
+				defer c.mu.Unlock()
 				logrus.Infof("Launching a new on demand Job for scan %v to run cis using profile %v", obj.Name, profile.Name)
-				configmaps, err := ciscore.NewConfigMaps(obj, profile, c.Name, c.ImageConfig)
+				benchmark, err := c.getClusterScanBenchmark(profile)
+				if err != nil {
+					v1.ClusterScanConditionReconciling.True(obj)
+					return objects, obj.Status, fmt.Errorf("Error when getting Benchmark: %v", err)
+				}
+				cmMap, err := ciscore.NewConfigMaps(obj, profile, benchmark, c.Name, c.ImageConfig, configmaps)
 				if err != nil {
 					v1.ClusterScanConditionReconciling.True(obj)
-					c.mu.Unlock()
 					return objects, obj.Status, fmt.Errorf("Error when creating ConfigMaps: %v", err)
 
 				}
 				service, err := ciscore.NewService(obj, profile, c.Name)
 				if err != nil {
 					v1.ClusterScanConditionReconciling.True(obj)
-					c.mu.Unlock()
 					return objects, obj.Status, fmt.Errorf("Error when creating Service: %v", err)
 				}
-
-				objects = append(objects, cisjob.New(obj, profile, c.Name, c.ImageConfig), configmaps[0], configmaps[1], configmaps[2], service)
+				objects = append(objects, cisjob.New(obj, profile, c.Name, c.ImageConfig), cmMap["configcm"], cmMap["plugincm"], cmMap["skipConfigcm"], service)
 
 				if v1.ClusterScanConditionFailed.IsTrue(obj) {
 					//clear the earlier failed status
@@ -126,7 +129,6 @@ func (c *Controller) handleClusterScans(ctx context.Context) error {
 				v1.ClusterScanConditionRunCompleted.Unknown(obj)
 				v1.ClusterScanConditionRunCompleted.Message(obj, "Creating Job to run the CIS scan")
 				c.setClusterScanStatusDisplay(obj)
-				c.mu.Unlock()
 				return objects, obj.Status, nil
 			}
 			return objects, obj.Status, nil
@@ -163,6 +165,11 @@ func (c *Controller) getClusterScanProfile(scan *v1.ClusterScan) (*v1.ClusterSca
 	return profile, nil
 }
 
+func (c *Controller) getClusterScanBenchmark(profile *v1.ClusterScanProfile) (*v1.ClusterScanBenchmark, error) {
+	clusterscanbmks := c.cisFactory.Cis().V1().ClusterScanBenchmark()
+	return clusterscanbmks.Get(profile.Spec.BenchmarkVersion, metav1.GetOptions{})
+}
+
 func (c *Controller) getDefaultClusterScanProfile(clusterprovider string) (string, error) {
 	var err error
 	configmaps := c.coreFactory.Core().V1().ConfigMap()