Merge pull request #52 from prachidamle/alert_metrics

CIS v2 Alerting: Adding Prometheus Metric Exporter for CIS, also adding ClusterScanAlertRule to the scan spec

Author: prachidamle

crds/clusterscan.yaml

@@ -43,14 +43,26 @@ spec:
       properties:
         spec:
           properties:
-            cronSchedule:
-              nullable: true
-              type: string
-            retentionCount:
-              type: integer
             scanProfileName:
               nullable: true
               type: string
+            scheduledScanConfig:
+              nullable: true
+              properties:
+                cronSchedule:
+                  nullable: true
+                  type: string
+                retentionCount:
+                  type: integer
+                scanAlertRule:
+                  nullable: true
+                  properties:
+                    alertOnComplete:
+                      type: boolean
+                    alertOnFailure:
+                      type: boolean
+                  type: object
+              type: object
             scoreWarning:
               enum:
               - pass
@@ -63,6 +75,9 @@ spec:
             NextScanAt:
               nullable: true
               type: string
+            ScanAlertingRuleName:
+              nullable: true
+              type: string
             conditions:
               items:
                 properties:

go.mod

@@ -2,19 +2,23 @@ module github.com/rancher/cis-operator
 
 go 1.13
 
-replace k8s.io/client-go => k8s.io/client-go v0.18.0
+replace k8s.io/client-go => k8s.io/client-go v0.19.2
 
 require (
 	github.com/blang/semver v3.5.0+incompatible
+	github.com/prometheus-operator/prometheus-operator v0.43.2
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.43.0
+	github.com/prometheus/client_golang v1.8.0
 	github.com/rancher/kubernetes-provider-detector v0.0.0-20200807181951-690274ab1fb3
 	github.com/rancher/lasso v0.0.0-20200820172840-0e4cc0ef5cb0
 	github.com/rancher/security-scan v0.2.2-0.20201117171930-af478b83fbe4
 	github.com/rancher/wrangler v0.6.2-0.20200829053106-7e1dd4260224
 	github.com/robfig/cron v1.2.0
-	github.com/sirupsen/logrus v1.4.2
+	github.com/sirupsen/logrus v1.6.0
 	github.com/urfave/cli v1.22.2
-	k8s.io/api v0.18.8
-	k8s.io/apiextensions-apiserver v0.18.0
-	k8s.io/apimachinery v0.18.8
-	k8s.io/client-go v10.0.0+incompatible
+	golang.org/x/tools v0.0.0-20201120032337-6d151481565c // indirect
+	k8s.io/api v0.19.2
+	k8s.io/apiextensions-apiserver v0.19.2
+	k8s.io/apimachinery v0.19.2
+	k8s.io/client-go v12.0.0+incompatible
 )

go.sum

@@ -16,6 +16,11 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 
+	"log"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	cisoperatorapiv1 "github.com/rancher/cis-operator/pkg/apis/cis.cattle.io/v1"
 	cisoperator "github.com/rancher/cis-operator/pkg/securityscan"
 )
@@ -26,8 +31,11 @@ var (
 	kubeConfig           string
 	threads              int
 	name                 string
-	securityScanImage    = "prachidamle/security-scan"
-	securityScanImageTag = "v0.1.20"
+	metricsPort          string
+	alertSeverity        string
+	debug                bool
+	securityScanImage    = "rancher/security-scan"
+	securityScanImageTag = "v0.2.1"
 	sonobuoyImage        = "rancher/sonobuoy-sonobuoy"
 	sonobuoyImageTag     = "v0.16.3"
 )
@@ -79,6 +87,23 @@ func main() {
 			Value:       "v0.16.3",
 			Destination: &sonobuoyImageTag,
 		},
+		cli.StringFlag{
+			Name:        "cis_metrics_port",
+			EnvVar:      "CIS_METRICS_PORT",
+			Value:       "8080",
+			Destination: &metricsPort,
+		},
+		cli.BoolFlag{
+			Name:        "debug",
+			EnvVar:      "CIS_OPERATOR_DEBUG",
+			Destination: &debug,
+		},
+		cli.StringFlag{
+			Name:        "alertSeverity",
+			EnvVar:      "CIS_ALERTS_SEVERITY",
+			Value:       "warning",
+			Destination: &alertSeverity,
+		},
 	}
 	app.Action = run
 
@@ -88,10 +113,11 @@ func main() {
 }
 
 func run(c *cli.Context) {
-
 	logrus.Info("Starting CIS-Operator")
 	ctx := signals.SetupSignalHandler(context.Background())
-
+	if debug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
 	kubeConfig = c.String("kubeconfig")
 	threads = c.Int("threads")
 	securityScanImage = c.String("security-scan-image")
@@ -110,6 +136,7 @@ func run(c *cli.Context) {
 		SecurityScanImageTag: securityScanImageTag,
 		SonobuoyImage:        sonobuoyImage,
 		SonobuoyImageTag:     sonobuoyImageTag,
+		AlertSeverity:        alertSeverity,
 	}
 
 	if err := validateConfig(imgConfig); err != nil {
@@ -124,6 +151,9 @@ func run(c *cli.Context) {
 	if err := ctl.Start(ctx, threads, 2*time.Hour); err != nil {
 		logrus.Fatalf("Error starting: %v", err)
 	}
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(":"+metricsPort, nil))
+
 	<-ctx.Done()
 	logrus.Info("Registered CIS controller")
 }
@@ -132,10 +162,8 @@ func validateConfig(imgConfig *cisoperatorapiv1.ScanImageConfig) error {
 	if imgConfig.SecurityScanImage == "" {
 		return errors.New("No Security-Scan Image specified")
 	}
-
 	if imgConfig.SonobuoyImage == "" {
 		return errors.New("No Sonobuoy tool Image specified")
 	}
-
 	return nil
 }

main.go

@@ -53,10 +53,8 @@ type ClusterScan struct {
 type ClusterScanSpec struct {
 	// scan profile to use
 	ScanProfileName string `json:"scanProfileName,omitempty"`
-	// Cron Expression for Schedule
-	CronSchedule string `yaml:"cron_schedule" json:"cronSchedule,omitempty"`
-	// Number of past scans to keep
-	RetentionCount int `yaml:"retentionCount" json:"retentionCount,omitempty"`
+	//config for scheduled scan
+	ScheduledScanConfig *ScheduledScanConfig `yaml:"scheduled_scan_config" json:"scheduledScanConfig,omitempty"`
 	// Specify if tests with "warn" output should be counted towards scan failure
 	ScoreWarning string `yaml:"score_warning" json:"scoreWarning,omitempty"`
 }
@@ -69,6 +67,7 @@ type ClusterScanStatus struct {
 	ObservedGeneration     int64                               `json:"observedGeneration"`
 	Conditions             []genericcondition.GenericCondition `json:"conditions,omitempty"`
 	NextScanAt             string                              `json:"NextScanAt"`
+	ScanAlertingRuleName   string                              `json:"ScanAlertingRuleName"`
 }
 
 type ClusterScanStatusDisplay struct {
@@ -87,6 +86,20 @@ type ClusterScanSummary struct {
 	NotApplicable int `json:"notApplicable"`
 }
 
+type ScheduledScanConfig struct {
+	// Cron Expression for Schedule
+	CronSchedule string `yaml:"cron_schedule" json:"cronSchedule,omitempty"`
+	// Number of past scans to keep
+	RetentionCount int `yaml:"retentionCount" json:"retentionCount,omitempty"`
+	//configure the alerts to be sent out
+	ScanAlertRule *ClusterScanAlertRule `json:"scanAlertRule,omitempty"`
+}
+
+type ClusterScanAlertRule struct {
+	AlertOnComplete bool `json:"alertOnComplete,omitempty"`
+	AlertOnFailure  bool `json:"alertOnFailure,omitempty"`
+}
+
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -145,4 +158,5 @@ type ScanImageConfig struct {
 	SecurityScanImageTag string
 	SonobuoyImage        string
 	SonobuoyImageTag     string
+	AlertSeverity        string
 }

pkg/apis/cis.cattle.io/v1/types.go

@@ -0,0 +1,73 @@
+package alert
+
+import (
+	"bytes"
+	"fmt"
+	"text/template"
+
+	meta1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	k8Yaml "k8s.io/apimachinery/pkg/util/yaml"
+
+	cisoperatorapiv1 "github.com/rancher/cis-operator/pkg/apis/cis.cattle.io/v1"
+	"github.com/rancher/wrangler/pkg/name"
+)
+
+const templateName = "prometheusrule.template"
+const templatePath = "./pkg/securityscan/alert/templates/prometheusrule.template"
+
+func NewPrometheusRule(clusterscan *cisoperatorapiv1.ClusterScan, clusterscanprofile *cisoperatorapiv1.ClusterScanProfile, imageConfig *cisoperatorapiv1.ScanImageConfig) (*monitoringv1.PrometheusRule, error) {
+	configdata := map[string]interface{}{
+		"namespace":       cisoperatorapiv1.ClusterScanNS,
+		"name":            name.SafeConcatName("rancher-cis-alerts", clusterscan.Name),
+		"severity":        imageConfig.AlertSeverity,
+		"scanName":        clusterscan.Name,
+		"scanProfileName": clusterscanprofile.Name,
+		"alertOnFailure":  clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnFailure,
+		"alertOnComplete": clusterscan.Spec.ScheduledScanConfig.ScanAlertRule.AlertOnComplete,
+	}
+	scanAlertRule, err := generatePrometheusRule(clusterscan, templateName, templatePath, configdata)
+	if err != nil {
+		return scanAlertRule, err
+	}
+
+	return scanAlertRule, nil
+}
+
+func generatePrometheusRule(clusterscan *cisoperatorapiv1.ClusterScan, templateName string, templateFile string, data map[string]interface{}) (*monitoringv1.PrometheusRule, error) {
+	scanAlertRule := &monitoringv1.PrometheusRule{}
+	obj, err := parseTemplate(clusterscan, templateName, templateFile, data)
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing the template %v", err)
+	}
+
+	if err := obj.Decode(&scanAlertRule); err != nil {
+		return nil, fmt.Errorf("Error decoding to template %v", err)
+	}
+
+	ownerRef := meta1.OwnerReference{
+		APIVersion: "cis.cattle.io/v1",
+		Kind:       "ClusterScan",
+		Name:       clusterscan.Name,
+		UID:        clusterscan.GetUID(),
+	}
+	scanAlertRule.ObjectMeta.OwnerReferences = append(scanAlertRule.ObjectMeta.OwnerReferences, ownerRef)
+
+	return scanAlertRule, nil
+}
+
+func parseTemplate(clusterscan *cisoperatorapiv1.ClusterScan, templateName string, templateFile string, data map[string]interface{}) (*k8Yaml.YAMLOrJSONDecoder, error) {
+	cmTemplate, err := template.New(templateName).ParseFiles(templateFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var b bytes.Buffer
+	err = cmTemplate.Execute(&b, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return k8Yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(b.String())), 1000), nil
+}

pkg/apis/cis.cattle.io/v1/zz_generated_deepcopy.go

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .name }}
+  namespace: {{ .namespace }}
+  labels:
+    app: rancher-monitoring
+spec:
+  groups:
+  - name: rancher-cis-scan-exporter
+    rules:
+{{- if .alertOnFailure }}
+    - alert: CISScanHasFailures
+      annotations:
+        description: CIS ClusterScan "{{ .scanName }}" has {{ "{{ $value }}" }} test failures
+        summary: CIS ClusterScan has tests failures
+      expr: cis_scan_num_tests_fail{scan_name="{{ .scanName }}"} > 0
+      for: 1m
+      labels:
+        severity: {{ .severity }}
+        job: rancher-cis-scan
+{{- end }}
+{{- if .alertOnComplete }}
+    - alert: CISScanHasCompleted
+      annotations:
+        description: CIS ClusterScan "{{ .scanName }}" with Cluster Scan profile  "{{ .scanProfileName }}" has completed.
+        summary: CIS ClusterScan has completed
+      expr: increase(cis_scan_num_scans_complete{scan_name="{{ .scanName }}"}[5m]) > 0
+      for: 1m
+      labels:
+        severity: {{ .severity }}
+        job: rancher-cis-scan
+{{- end }}