Merge pull request #58 from myaser/histogram

migrate to histogram

Author: zbindenren

README.md

@@ -209,4 +209,4 @@ All performed checks expose metrics which can be used to monitor/alert:
 
 At `/metrics` you will find these:
 - `kubenurse_errors_total`: Kubenurse error counter partitioned by error type
-- `kubenurse_request_duration`: Kubenurse request duration partitioned by error type, summary over one minute
+- `kubenurse_request_duration`: a histogram for Kubenurse request duration partitioned by error type

doc/grafana-kubenurse.json

@@ -66,7 +66,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "kubenurse_request_duration{quantile=\"0.9\", type=~\"$type\"}",
+          "expr": "histogram_quantile(0.9, rate(kubenurse_request_duration{type=~\"$type\"}[1m]))",
           "interval": "",
           "legendFormat": "{{pod}}: {{type}}",
           "refId": "A"

internal/servicecheck/servicecheck.go

@@ -33,18 +33,17 @@ func New(ctx context.Context, discovery *kubediscovery.Client, promRegistry *pro
 		[]string{"type"},
 	)
 
-	durationSummary := prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace:  metricsNamespace,
-			Name:       "request_duration",
-			Help:       "Kubenurse request duration partitioned by error type",
-			MaxAge:     1 * time.Minute,
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	durationHistogram := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Name:      "request_duration",
+			Help:      "Kubenurse request duration partitioned by target path",
+			Buckets:   prometheus.DefBuckets,
 		},
 		[]string{"type"},
 	)
 
-	promRegistry.MustRegister(errorCounter, durationSummary)
+	promRegistry.MustRegister(errorCounter, durationHistogram)
 
 	// setup http transport
 	transport, err := generateRoundTripper(os.Getenv("KUBENURSE_EXTRA_CA"), os.Getenv("KUBENURSE_INSECURE") == "true")
@@ -65,7 +64,7 @@ func New(ctx context.Context, discovery *kubediscovery.Client, promRegistry *pro
 		httpClient:         httpClient,
 		cacheTTL:           cacheTTL,
 		errorCounter:       errorCounter,
-		durationSummary:    durationSummary,
+		durationHistogram:  durationHistogram,
 		stop:               make(chan struct{}),
 	}, nil
 }
@@ -210,7 +209,7 @@ func (c *Checker) measure(check Check, label string) (string, error) {
 	res, err := check()
 
 	// Process metrics
-	c.durationSummary.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	c.durationHistogram.WithLabelValues(label).Observe(time.Since(start).Seconds())
 
 	if err != nil {
 		log.Printf("failed request for %s with %v", label, err)

internal/servicecheck/types.go

@@ -34,8 +34,8 @@ type Checker struct {
 	discovery *kubediscovery.Client
 
 	// metrics
-	errorCounter    *prometheus.CounterVec
-	durationSummary *prometheus.SummaryVec
+	errorCounter      *prometheus.CounterVec
+	durationHistogram *prometheus.HistogramVec
 
 	// Http Client for https requests
 	httpClient *http.Client