Merge pull request #72 from prachidamle/fix_custom

prachidamle · web-flow · commit 05cb861f6405 · 2020-12-11T13:29:54.000-08:00
CIS v2 Alerting and Custom Benchmark fixes
diff --git a/crds/clusterscan.yaml b/crds/clusterscan.yaml
@@ -28,7 +28,7 @@ spec:
   - JSONPath: .status.lastRunTimestamp
     name: LastRunTimestamp
     type: string
-  - JSONPath: .spec.cronSchedule
+  - JSONPath: .spec.scheduledScanConfig.cronSchedule
     name: CronSchedule
     type: string
   group: cis.cattle.io
diff --git a/crds/clusterscanbenchmark.yaml b/crds/clusterscanbenchmark.yaml
@@ -13,6 +13,12 @@ spec:
   - JSONPath: .spec.maxKubernetesVersion
     name: MaxKubernetesVersion
     type: string
+  - JSONPath: .spec.customBenchmarkConfigMapName
+    name: customBenchmarkConfigMapName
+    type: string
+  - JSONPath: .spec.customBenchmarkConfigMapNamespace
+    name: customBenchmarkConfigMapNamespace
+    type: string
   group: cis.cattle.io
   names:
     kind: ClusterScanBenchmark
@@ -31,7 +37,7 @@ spec:
             customBenchmarkConfigMapName:
               nullable: true
               type: string
-            customBenchmarkConfigMapNameSpace:
+            customBenchmarkConfigMapNamespace:
               nullable: true
               type: string
             maxKubernetesVersion:
diff --git a/examples/clusterscanrke.yml b/examples/clusterscanrke.yml
@@ -2,6 +2,11 @@
 apiVersion: cis.cattle.io/v1
 kind: ClusterScan
 metadata:
-  name: rke-cis
+  name: can-you-alert
 spec:
   scanProfileName: rke-profile-hardened
+  scheduledScanConfig:
+    cronSchedule: "*/2 * * * *"
+    scanAlertRule:
+      alertOnComplete: true
+      alertOnFailure: true
diff --git a/examples/custombenchmarkscan.yml b/examples/custombenchmarkscan.yml
@@ -0,0 +1,7 @@
+---
+apiVersion: cis.cattle.io/v1
+kind: ClusterScan
+metadata:
+  name: rke-cis-1.6-perm-custom
+spec:
+  scanProfileName: "rke-cis-1.6-permissive"
diff --git a/main.go b/main.go
@@ -38,6 +38,7 @@ var (
 	securityScanImageTag = "v0.2.1"
 	sonobuoyImage        = "rancher/sonobuoy-sonobuoy"
 	sonobuoyImageTag     = "v0.16.3"
+	clusterName          string
 )
 
 func main() {
@@ -104,6 +105,12 @@ func main() {
 			Value:       "warning",
 			Destination: &alertSeverity,
 		},
+		cli.StringFlag{
+			Name:        "clusterName",
+			EnvVar:      "CLUSTER_NAME",
+			Value:       "",
+			Destination: &clusterName,
+		},
 	}
 	app.Action = run
 
@@ -137,6 +144,7 @@ func run(c *cli.Context) {
 		SonobuoyImage:        sonobuoyImage,
 		SonobuoyImageTag:     sonobuoyImageTag,
 		AlertSeverity:        alertSeverity,
+		ClusterName:          clusterName,
 	}
 
 	if err := validateConfig(imgConfig); err != nil {
diff --git a/pkg/apis/cis.cattle.io/v1/types.go b/pkg/apis/cis.cattle.io/v1/types.go
@@ -117,7 +117,7 @@ type ClusterScanBenchmarkSpec struct {
 	MaxKubernetesVersion string `json:"maxKubernetesVersion,omitempty"`
 
 	CustomBenchmarkConfigMapName      string `json:"customBenchmarkConfigMapName,omitempty"`
-	CustomBenchmarkConfigMapNameSpace string `json:"customBenchmarkConfigMapNameSpace,omitempty"`
+	CustomBenchmarkConfigMapNamespace string `json:"customBenchmarkConfigMapNamespace,omitempty"`
 }
 
 // +genclient
@@ -159,4 +159,5 @@ type ScanImageConfig struct {
 	SonobuoyImage        string
 	SonobuoyImageTag     string
 	AlertSeverity        string
+	ClusterName          string
 }
diff --git a/pkg/crds/crd.go b/pkg/crds/crd.go
@@ -49,7 +49,7 @@ func List() []crd.CRD {
 				WithColumn("Warn", ".status.summary.warn").
 				WithColumn("Not Applicable", ".status.summary.notApplicable").
 				WithColumn("LastRunTimestamp", ".status.lastRunTimestamp").
-				WithColumn("CronSchedule", ".spec.cronSchedule")
+				WithColumn("CronSchedule", ".spec.scheduledScanConfig.cronSchedule")
 		}),
 		newCRD(&cisoperator.ClusterScanProfile{}, func(c crd.CRD) crd.CRD {
 			return c.
@@ -64,7 +64,9 @@ func List() []crd.CRD {
 			return c.
 				WithColumn("ClusterProvider", ".spec.clusterProvider").
 				WithColumn("MinKubernetesVersion", ".spec.minKubernetesVersion").
-				WithColumn("MaxKubernetesVersion", ".spec.maxKubernetesVersion")
+				WithColumn("MaxKubernetesVersion", ".spec.maxKubernetesVersion").
+				WithColumn("customBenchmarkConfigMapName", ".spec.customBenchmarkConfigMapName").
+				WithColumn("customBenchmarkConfigMapNamespace", ".spec.customBenchmarkConfigMapNamespace")
 		}),
 	}
 }
diff --git a/pkg/securityscan/alert/prometheusrule.go b/pkg/securityscan/alert/prometheusrule.go
@@ -26,6 +26,7 @@ func NewPrometheusRule(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanpro
 		"scanProfileName": clusterscanprofile.Name,
 		"alertOnFailure":  clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnFailure,
 		"alertOnComplete": clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnComplete,
+		"failOnWarn":      clusterscan.Spec.ScoreWarning == cisoperatorapiv1.ClusterScanFailOnWarning,
 	}
 	scanAlertRule, err := generatePrometheusRule(clusterscan, templateName, templatePath, configdata)
 	if err != nil {
diff --git a/pkg/securityscan/alert/templates/prometheusrule.template b/pkg/securityscan/alert/templates/prometheusrule.template
@@ -12,9 +12,13 @@ spec:
 {{- if .alertOnFailure }}
     - alert: CISScanHasFailures
       annotations:
-        description: CIS ClusterScan "{{ .scanName }}" has {{ "{{ $value }}" }} test failures
+        description: CIS ClusterScan "{{ .scanName }}" has {{ "{{ $value }}" }} test failures or warnings
         summary: CIS ClusterScan has tests failures
+      {{- if .failOnWarn }}
+      expr: cis_scan_num_tests_fail{scan_name="{{ .scanName }}"} > 0 or ON(scan_name) cis_scan_num_tests_warn{scan_name="{{ .scanName }}"} > 0
+      {{- else }}
       expr: cis_scan_num_tests_fail{scan_name="{{ .scanName }}"} > 0
+      {{- end }}
       for: 1m
       labels:
         severity: {{ .severity }}
diff --git a/pkg/securityscan/controller.go b/pkg/securityscan/controller.go
@@ -50,6 +50,7 @@ type Controller struct {
 	numTestsTotal    *prometheus.GaugeVec
 	numTestsNA       *prometheus.GaugeVec
 	numTestsPassed   *prometheus.GaugeVec
+	numTestsWarn     *prometheus.GaugeVec
 }
 
 func NewController(ctx context.Context, cfg *rest.Config, namespace, name string, imgConfig *cisoperatorapiv1.ScanImageConfig) (ctl *Controller, err error) {
@@ -186,6 +187,7 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numTestsFailed); err != nil {
@@ -202,6 +204,7 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numScansComplete); err != nil {
@@ -218,6 +221,7 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numTestsTotal); err != nil {
@@ -234,6 +238,7 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numTestsPassed); err != nil {
@@ -250,6 +255,7 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numTestsSkipped); err != nil {
@@ -266,11 +272,29 @@ func initializeMetrics(ctl *Controller) error {
 			"scan_name",
 			// name of the clusterScanProfile used for scanning
 			"scan_profile_name",
+			"cluster_name",
 		},
 	)
 	if err := prometheus.Register(ctl.numTestsNA); err != nil {
 		return err
 	}
 
+	ctl.numTestsWarn = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "cis_scan_num_tests_warn",
+			Help: "Number of tests having warn status in the CIS scans, partioned by scan_name, scan_profile_name",
+		},
+		[]string{
+			// scan_name will be set to "manual" for on-demand manual scans and the actual name set for the scheduled scans
+			"scan_name",
+			// name of the clusterScanProfile used for scanning
+			"scan_profile_name",
+			"cluster_name",
+		},
+	)
+	if err := prometheus.Register(ctl.numTestsWarn); err != nil {
+		return err
+	}
+
 	return nil
 }
diff --git a/pkg/securityscan/core/configmap.go b/pkg/securityscan/core/configmap.go
@@ -146,5 +146,5 @@ func getCustomBenchmarkConfigMap(benchmark *cisoperatorapiv1.ClusterScanBenchmar
 	if benchmark.Spec.CustomBenchmarkConfigMapName == "" {
 		return nil, nil
 	}
-	return configmapsClient.Get(benchmark.Spec.CustomBenchmarkConfigMapNameSpace, benchmark.Spec.CustomBenchmarkConfigMapName, metav1.GetOptions{})
+	return configmapsClient.Get(benchmark.Spec.CustomBenchmarkConfigMapNamespace, benchmark.Spec.CustomBenchmarkConfigMapName, metav1.GetOptions{})
 }
diff --git a/pkg/securityscan/scanHandler.go b/pkg/securityscan/scanHandler.go
@@ -79,21 +79,6 @@ func (c *Controller) handleClusterScans(ctx context.Context) error {
 					return objects, obj.Status, nil
 				}
 
-				if obj.Spec.ScheduledScanConfig != nil && obj.Spec.ScheduledScanConfig.ScanAlertRule != nil {
-					if obj.Status.ScanAlertingRuleName == "" {
-						alertRule, err := cisalert.NewPrometheusRule(obj, profile, c.ImageConfig)
-						if err != nil {
-							v1.ClusterScanConditionReconciling.True(obj)
-							return objects, obj.Status, fmt.Errorf("Error when trying to create a PrometheusRule: %v", err)
-						}
-						ruleCreated, err := c.monitoringClient.PrometheusRules(v1.ClusterScanNS).Create(ctx, alertRule, metav1.CreateOptions{})
-						if err != nil {
-							v1.ClusterScanConditionReconciling.True(obj)
-							return objects, obj.Status, fmt.Errorf("Error when creating PrometheusRule: %v", err)
-						}
-						obj.Status.ScanAlertingRuleName = ruleCreated.Name
-					}
-				}
 				if err := c.isRunnerPodPresent(); err != nil {
 					return objects, obj.Status, fmt.Errorf("Retrying ClusterScan %v since got error: %v ", obj.Name, err)
 				}
@@ -119,6 +104,23 @@ func (c *Controller) handleClusterScans(ctx context.Context) error {
 				}
 				objects = append(objects, cisjob.New(obj, profile, c.Name, c.ImageConfig), cmMap["configcm"], cmMap["plugincm"], cmMap["skipConfigcm"], service)
 
+				if obj.Spec.ScheduledScanConfig != nil && obj.Spec.ScheduledScanConfig.ScanAlertRule != nil {
+					if obj.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnComplete || obj.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnFailure {
+						if obj.Status.ScanAlertingRuleName == "" {
+							alertRule, err := cisalert.NewPrometheusRule(obj, profile, c.ImageConfig)
+							if err != nil {
+								v1.ClusterScanConditionReconciling.True(obj)
+								return objects, obj.Status, fmt.Errorf("Error when trying to create a PrometheusRule: %v", err)
+							}
+							ruleCreated, err := c.monitoringClient.PrometheusRules(v1.ClusterScanNS).Create(ctx, alertRule, metav1.CreateOptions{})
+							if err != nil {
+								logrus.Errorf("Alerts will not be sent out for this scan %v due to this error when creating PrometheusRule: %v", obj.Name, err)
+							} else {
+								obj.Status.ScanAlertingRuleName = ruleCreated.Name
+							}
+						}
+					}
+				}
 				if v1.ClusterScanConditionFailed.IsTrue(obj) {
 					//clear the earlier failed status
 					v1.ClusterScanConditionFailed.False(obj)
diff --git a/pkg/securityscan/scanMetricsHandler.go b/pkg/securityscan/scanMetricsHandler.go
@@ -4,9 +4,10 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/sirupsen/logrus"
-
 	v1 "github.com/rancher/cis-operator/pkg/apis/cis.cattle.io/v1"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )
 
 func (c *Controller) handleClusterScanMetrics(ctx context.Context) error {
@@ -19,16 +20,7 @@ func (c *Controller) handleClusterScanMetrics(ctx context.Context) error {
 		if !(v1.ClusterScanConditionAlerted.IsUnknown(obj) && v1.ClusterScanConditionComplete.IsTrue(obj)) {
 			return obj, nil
 		}
-		if obj.Spec.ScheduledScanConfig != nil && obj.Spec.ScheduledScanConfig.ScanAlertRule == nil {
-			logrus.Debugf("No AlertRules configured for scan %v", obj.Name)
-			v1.ClusterScanConditionAlerted.False(obj)
-			v1.ClusterScanConditionAlerted.Message(obj, "No AlertRule configured for this scan")
-			_, err := scans.UpdateStatus(obj)
-			if err != nil {
-				return obj, fmt.Errorf("Retrying, got error %v in updating condition of scan object: %v ", err, obj.Name)
-			}
-			return obj, nil
-		}
+
 		logrus.Debugf("Updating metrics for scan %v", obj.Name)
 
 		scanName := "manual"
@@ -41,20 +33,49 @@ func (c *Controller) handleClusterScanMetrics(ctx context.Context) error {
 		numTestsNA := float64(obj.Status.Summary.NotApplicable)
 		numTestsSkip := float64(obj.Status.Summary.Skip)
 		numTestsPass := float64(obj.Status.Summary.Pass)
+		numTestsWarn := float64(obj.Status.Summary.Warn)
+		clusterName := c.ImageConfig.ClusterName
 
-		c.numTestsFailed.WithLabelValues(scanName, scanProfileName).Set(numTestsFailed)
-		c.numScansComplete.WithLabelValues(scanName, scanProfileName).Inc()
-		c.numTestsTotal.WithLabelValues(scanName, scanProfileName).Set(numTestsTotal)
-		c.numTestsPassed.WithLabelValues(scanName, scanProfileName).Set(numTestsPass)
-		c.numTestsSkipped.WithLabelValues(scanName, scanProfileName).Set(numTestsSkip)
-		c.numTestsNA.WithLabelValues(scanName, scanProfileName).Set(numTestsNA)
+		c.numTestsFailed.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsFailed)
+		c.numScansComplete.WithLabelValues(scanName, scanProfileName, clusterName).Inc()
+		c.numTestsTotal.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsTotal)
+		c.numTestsPassed.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsPass)
+		c.numTestsSkipped.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsSkip)
+		c.numTestsNA.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsNA)
+		c.numTestsWarn.WithLabelValues(scanName, scanProfileName, clusterName).Set(numTestsWarn)
 
 		logrus.Debugf("Done updating metrics for scan %v", obj.Name)
-		v1.ClusterScanConditionAlerted.True(obj)
-		_, err := scans.UpdateStatus(obj)
-		if err != nil {
-			return obj, fmt.Errorf("Retrying, got error %v in updating condition of scan object: %v ", err, obj.Name)
+
+		if obj.Spec.ScheduledScanConfig != nil {
+			updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+				var err error
+				scanObj, err := scans.Get(obj.Name, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+				if scanObj.Spec.ScheduledScanConfig.ScanAlertRule == nil ||
+					(scanObj.Spec.ScheduledScanConfig.ScanAlertRule != nil &&
+						!scanObj.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnComplete &&
+						!scanObj.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnFailure) {
+					logrus.Debugf("No AlertRules configured for scan %v", scanObj.Name)
+					v1.ClusterScanConditionAlerted.False(scanObj)
+					v1.ClusterScanConditionAlerted.Message(scanObj, "No AlertRule configured for this scan")
+				} else if scanObj.Status.ScanAlertingRuleName == "" {
+					logrus.Debugf("Error creating PrometheusRule for scan %v", scanObj.Name)
+					v1.ClusterScanConditionAlerted.False(scanObj)
+					v1.ClusterScanConditionAlerted.Message(scanObj, "Alerts will not work due to the error creating PrometheusRule, Please check if Monitoring app is installed")
+				} else {
+					v1.ClusterScanConditionAlerted.True(scanObj)
+				}
+				_, err = scans.UpdateStatus(scanObj)
+				return err
+			})
+
+			if updateErr != nil {
+				return obj, fmt.Errorf("Retrying, got error %v in updating condition of scan object: %v ", updateErr, obj.Name)
+			}
 		}
+
 		return obj, nil
 	})
 	return nil

Original file line number	Diff line number	Diff line change
`@@ -117,7 +117,7 @@ type ClusterScanBenchmarkSpec struct {`
`117`	`117`	MaxKubernetesVersion string `json:"maxKubernetesVersion,omitempty"`
`118`	`118`
`119`	`119`	CustomBenchmarkConfigMapName string `json:"customBenchmarkConfigMapName,omitempty"`
`120`		- CustomBenchmarkConfigMapNameSpace string `json:"customBenchmarkConfigMapNameSpace,omitempty"`
	`120`	+ CustomBenchmarkConfigMapNamespace string `json:"customBenchmarkConfigMapNamespace,omitempty"`
`121`	`121`	`}`
`122`	`122`
`123`	`123`	`// +genclient`
`@@ -159,4 +159,5 @@ type ScanImageConfig struct {`
`159`	`159`	`SonobuoyImage string`
`160`	`160`	`SonobuoyImageTag string`
`161`	`161`	`AlertSeverity string`
	`162`	`+ ClusterName string`
`162`	`163`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,7 @@ func NewPrometheusRule(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanpro`
`26`	`26`	`"scanProfileName": clusterscanprofile.Name,`
`27`	`27`	`"alertOnFailure": clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnFailure,`
`28`	`28`	`"alertOnComplete": clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnComplete,`
	`29`	`+ "failOnWarn": clusterscan.Spec.ScoreWarning == cisoperatorapiv1.ClusterScanFailOnWarning,`
`29`	`30`	`}`
`30`	`31`	`scanAlertRule, err := generatePrometheusRule(clusterscan, templateName, templatePath, configdata)`
`31`	`32`	`if err != nil {`