Merge pull request #595 from kartverket/customizable-scrape-interval

prometheus: add scrapeInterval

Author: evenh

api/v1alpha1/application_types.go

@@ -3,14 +3,15 @@ package v1alpha1
 import (
 	"encoding/json"
 	"errors"
+	"time"
+
 	"github.com/kartverket/skiperator/api/v1alpha1/digdirator"
 	"github.com/kartverket/skiperator/api/v1alpha1/istiotypes"
 	"github.com/kartverket/skiperator/api/v1alpha1/podtypes"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"time"
 )
 
 // +kubebuilder:object:root=true
@@ -333,6 +334,16 @@ type PrometheusConfig struct {
 	//+kubebuilder:default:=false
 	//+kubebuilder:validation:Optional
 	AllowAllMetrics bool `json:"allowAllMetrics,omitempty"`
+
+	// ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+	// The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+	// If minutes ("Xm") are used, the value must be at least 1m.
+	//
+	//+kubebuilder:default:="60s"
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:validation:XValidation:rule="self == '' || self.matches('^([0-9]+[sm])+$')",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be empty (default applies) or in the format of <number>s or <number>m.'"
+	//+kubebuilder:validation:XValidation:rule="self == '' || (self.endsWith('m') && int(self.split('m')[0]) >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >= 15 && int(self.split('s')[0]) % 5 == 0)",messageExpression="'Rejected: ' + self + ' as an invalid value. ScrapeInterval must be at least 15s (if using <s>) and divisible by 5, or at least 1m (if using <m>).'"
+	ScrapeInterval string `json:"scrapeInterval,omitempty"`
 }
 
 func NewDefaultReplicas() Replicas {

cmd/skiperator/main.go

@@ -55,6 +55,7 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
 		Development: !*isDeployment,
 		Level:       parsedLogLevel,
+		DestWriter:  os.Stdout,
 	})))
 
 	setupLog.Info(fmt.Sprintf("Running skiperator %s (commit %s)", Version, Commit))

config/crd/skiperator.kartverket.no_applications.yaml

@@ -947,6 +947,24 @@ spec:
                     description: The port number or name where metrics are exposed
                       (at the Pod level).
                     x-kubernetes-int-or-string: true
+                  scrapeInterval:
+                    default: 60s
+                    description: |-
+                      ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+                      The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+                      If minutes ("Xm") are used, the value must be at least 1m.
+                    type: string
+                    x-kubernetes-validations:
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be empty (default applies) or in
+                        the format of <number>s or <number>m.'''
+                      rule: self == '' || self.matches('^([0-9]+[sm])+$')
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be at least 15s (if using <s>)
+                        and divisible by 5, or at least 1m (if using <m>).'''
+                      rule: self == '' || (self.endsWith('m') && int(self.split('m')[0])
+                        >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >=
+                        15 && int(self.split('s')[0]) % 5 == 0)
                 required:
                 - port
                 type: object

config/crd/skiperator.kartverket.no_skipjobs.yaml

@@ -880,6 +880,24 @@ spec:
                     description: The port number or name where metrics are exposed
                       (at the Pod level).
                     x-kubernetes-int-or-string: true
+                  scrapeInterval:
+                    default: 60s
+                    description: |-
+                      ScrapeInterval specifies the interval at which Prometheus should scrape the metrics.
+                      The interval must be at least 15 seconds (if using "Xs") and divisible by 5.
+                      If minutes ("Xm") are used, the value must be at least 1m.
+                    type: string
+                    x-kubernetes-validations:
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be empty (default applies) or in
+                        the format of <number>s or <number>m.'''
+                      rule: self == '' || self.matches('^([0-9]+[sm])+$')
+                    - messageExpression: '''Rejected: '' + self + '' as an invalid
+                        value. ScrapeInterval must be at least 15s (if using <s>)
+                        and divisible by 5, or at least 1m (if using <m>).'''
+                      rule: self == '' || (self.endsWith('m') && int(self.split('m')[0])
+                        >= 1) || (self.endsWith('s') && int(self.split('s')[0]) >=
+                        15 && int(self.split('s')[0]) % 5 == 0)
                 required:
                 - port
                 type: object

internal/controllers/application.go

@@ -24,10 +24,10 @@ import (
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/maskinporten"
 	networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/pdb"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/service"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount"
-	"github.com/kartverket/skiperator/pkg/resourcegenerator/servicemonitor"
 	"github.com/kartverket/skiperator/pkg/util"
 	nais_io_v1 "github.com/nais/liberator/pkg/apis/nais.io/v1"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -207,7 +207,7 @@ func (r *ApplicationReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		networkpolicy.Generate,
 		authorizationpolicy.Generate,
 		pdb.Generate,
-		servicemonitor.Generate,
+		prometheus.Generate,
 		idporten.Generate,
 		maskinporten.Generate,
 		deployment.Generate,

internal/controllers/skipjob.go

@@ -14,7 +14,7 @@ import (
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/istio/telemetry"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/job"
 	networkpolicy "github.com/kartverket/skiperator/pkg/resourcegenerator/networkpolicy/dynamic"
-	"github.com/kartverket/skiperator/pkg/resourcegenerator/podmonitor"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/prometheus"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/serviceaccount"
 	istionetworkingv1 "istio.io/client-go/pkg/apis/networking/v1"
@@ -163,7 +163,7 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		serviceentry.Generate,
 		auth.Generate,
 		job.Generate,
-		podmonitor.Generate,
+		prometheus.Generate,
 		telemetry.Generate,
 	}
 

pkg/resourcegenerator/podmonitor/pod_monitor.go pkg/resourcegenerator/prometheus/pod_monitor.go

@@ -1,16 +1,21 @@
-package podmonitor
+package prometheus
 
 import (
 	"fmt"
+	"strings"
+
 	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
 	"github.com/kartverket/skiperator/pkg/reconciliation"
 	"github.com/kartverket/skiperator/pkg/util"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"strings"
 )
 
-func Generate(r reconciliation.Reconciliation) error {
+func init() {
+	multiGenerator.Register(reconciliation.JobType, generateForSkipJob)
+}
+
+func generateForSkipJob(r reconciliation.Reconciliation) error {
 	ctxLog := r.GetLogger()
 	ctxLog.Debug("Attempting to generate podmonitor for skipjob", "skipjob", r.GetSKIPObject().GetName())
 
@@ -41,6 +46,7 @@ func Generate(r reconciliation.Reconciliation) error {
 			{
 				Path:       util.IstioMetricsPath,
 				TargetPort: &util.IstioMetricsPortName,
+				Interval:   getScrapeInterval(skipJob.Spec.Prometheus),
 			},
 		},
 	}

pkg/resourcegenerator/prometheus/prometheus.go

@@ -0,0 +1,25 @@
+package prometheus
+
+import (
+	"github.com/kartverket/skiperator/api/v1alpha1"
+	"github.com/kartverket/skiperator/pkg/reconciliation"
+	"github.com/kartverket/skiperator/pkg/resourcegenerator/resourceutils/generator"
+	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+)
+
+var (
+	multiGenerator        = generator.NewMulti()
+	defaultScrapeInterval = pov1.Duration("60s")
+)
+
+func Generate(r reconciliation.Reconciliation) error {
+	return multiGenerator.Generate(r, "PrometheusCRD")
+}
+
+func getScrapeInterval(pc *v1alpha1.PrometheusConfig) pov1.Duration {
+	if pc == nil {
+		return defaultScrapeInterval
+	}
+
+	return pov1.Duration(pc.ScrapeInterval)
+}

pkg/resourcegenerator/servicemonitor/service_monitor.go pkg/resourcegenerator/prometheus/service_monitor.go

@@ -1,16 +1,21 @@
-package servicemonitor
+package prometheus
 
 import (
 	"fmt"
+	"strings"
+
 	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
 	"github.com/kartverket/skiperator/pkg/reconciliation"
 	"github.com/kartverket/skiperator/pkg/util"
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"strings"
 )
 
-func Generate(r reconciliation.Reconciliation) error {
+func init() {
+	multiGenerator.Register(reconciliation.ApplicationType, generateForApplication)
+}
+
+func generateForApplication(r reconciliation.Reconciliation) error {
 	ctxLog := r.GetLogger()
 	if r.GetType() != reconciliation.ApplicationType {
 		return fmt.Errorf("unsupported type %s in service monitor", r.GetType())
@@ -44,6 +49,7 @@ func Generate(r reconciliation.Reconciliation) error {
 			{
 				Path:       util.IstioMetricsPath,
 				TargetPort: &util.IstioMetricsPortName,
+				Interval:   getScrapeInterval(application.Spec.Prometheus),
 				MetricRelabelConfigs: []pov1.RelabelConfig{
 					{
 						Action:       "drop",

tests/application/service-monitor/application-istio-assert.yaml

@@ -68,6 +68,7 @@ spec:
   endpoints:
     - targetPort: istio-metrics
       path: /stats/prometheus
+      interval: "60s"
       metricRelabelings:
       - action: drop
         regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket

tests/application/service-monitor/application-simple-assert.yaml

@@ -57,6 +57,7 @@ spec:
   endpoints:
     - targetPort: istio-metrics
       path: /stats/prometheus
+      interval: "60s"
       metricRelabelings:
       - action: drop
         regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket
@@ -100,4 +101,4 @@ spec:
               app.kubernetes.io/instance: alloy
               app.kubernetes.io/name: alloy
       ports:
-        - port: istio-metrics
+        - port: istio-metrics

tests/application/service-monitor/application-simple-custom-interval-assert.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    instance: primary
+  name: some-simple-monitored-app-1
+  namespace: sm-istio-ns-2
+spec:
+  endpoints:
+    - targetPort: istio-metrics
+      path: /stats/prometheus
+      interval: "90s"
+      metricRelabelings:
+      - action: drop
+        regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket
+        sourceLabels:
+        - __name__
+  selector:
+    matchLabels:
+      app: some-simple-monitored-app-1
+  namespaceSelector:
+    matchNames:
+      - sm-istio-ns-2

tests/application/service-monitor/application-simple-custom-interval-invalid.yaml

@@ -0,0 +1,11 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: Application
+metadata:
+  name: some-simple-monitored-app-1
+  namespace: sm-istio-ns-2
+spec:
+  image: image
+  port: 8080
+  prometheus:
+    port: 8080
+    scrapeInterval: 10s

tests/application/service-monitor/application-simple-custom-interval.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sm-istio-ns-2
+  labels:
+    istio.io/rev: "revision-1"
+---
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: Application
+metadata:
+  name: some-simple-monitored-app-1
+  namespace: sm-istio-ns-2
+spec:
+  image: image
+  port: 8080
+  prometheus:
+    port: 8080
+    scrapeInterval: 90s

tests/application/service-monitor/chainsaw-test.yaml

@@ -23,3 +23,10 @@ spec:
             file: patch-application-allowall.yaml
         - assert:
             file: patch-application-allowall-assert.yaml
+    - try:
+        - apply:
+            file: application-simple-custom-interval.yaml
+        - assert:
+            file: application-simple-custom-interval-assert.yaml
+        - error:
+            file: application-simple-custom-interval-invalid.yaml

tests/application/service-monitor/patch-application-allowall-assert.yaml

@@ -9,6 +9,7 @@ spec:
   endpoints:
     - targetPort: istio-metrics
       path: /stats/prometheus
+      interval: "60s"
   selector:
     matchLabels:
       app: some-monitored-app-1

tests/skipjob/podmonitor/chainsaw-test.yaml

@@ -18,6 +18,11 @@ spec:
             file: patch-skipjob-with-monitoring.yaml
         - assert:
             file: patch-skipjob-with-monitoring-assert.yaml
+    - try:
+        - patch:
+            file: patch-skipjob-with-monitoring-custom-interval.yaml
+        - assert:
+            file: patch-skipjob-with-monitoring-custom-interval-assert.yaml
     - try:
         - patch:
             file: patch-skipjob-remove-monitoring.yaml

tests/skipjob/podmonitor/patch-skipjob-remove-monitoring-error.yaml

@@ -11,6 +11,7 @@ spec:
   podMetricsEndpoints:
     - targetPort: istio-metrics
       path: "/stats/prometheus"
+      interval: "60s"
       metricRelabelings:
         - action: drop
           regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket

tests/skipjob/podmonitor/patch-skipjob-with-monitoring-assert.yaml

@@ -11,6 +11,7 @@ spec:
   podMetricsEndpoints:
     - targetPort: istio-metrics
       path: "/stats/prometheus"
+      interval: "60s"
   selector:
     matchLabels:
       app: podmonitor

tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval-assert.yaml

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    instance: primary
+  name: podmonitor-monitor
+spec:
+  namespaceSelector:
+    matchNames:
+      - podmonitor-ns
+  podMetricsEndpoints:
+    - targetPort: istio-metrics
+      path: "/stats/prometheus"
+      interval: "100s"
+  selector:
+    matchLabels:
+      app: podmonitor

tests/skipjob/podmonitor/patch-skipjob-with-monitoring-custom-interval.yaml

@@ -0,0 +1,17 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: SKIPJob
+metadata:
+  name: podmonitor
+spec:
+  container:
+    image: "perl:5.34.0"
+    command:
+      - "perl"
+      - "-Mbignum=bpi"
+      - "-wle"
+      - "print bpi(2000)"
+  prometheus:
+    path: /metrics
+    port: 8080
+    allowAllMetrics: true
+    scrapeInterval: "100s"

tests/skipjob/podmonitor/skipjob-with-monitoring-assert.yaml

@@ -55,6 +55,7 @@ spec:
   podMetricsEndpoints:
     - targetPort: istio-metrics
       path: "/stats/prometheus"
+      interval: "60s"
       metricRelabelings:
         - action: drop
           regex: istio_request_bytes_bucket|istio_response_bytes_bucket|istio_request_duration_milliseconds_bucket