Add histogram for health transmission duration (#147)

* Add histogram for health transmission duration

* Add buckets

* One decimal place

Author: cristinaleonr

cmd/heartbeat/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -19,6 +20,7 @@ import (
 	"github.com/m-lab/locate/cmd/heartbeat/health"
 	"github.com/m-lab/locate/cmd/heartbeat/registration"
 	"github.com/m-lab/locate/connection"
+	"github.com/m-lab/locate/metrics"
 	"github.com/m-lab/locate/static"
 )
 
@@ -124,10 +126,15 @@ func write(ws *connection.Conn, hc *health.Checker, ldr *registration.Loader) {
 				log.Printf("updated registration to %v", reg)
 			}
 		case <-hbTicker.C:
+			t := time.Now()
 			score := getHealth(hc)
 			healthMsg := v2.Health{Score: score}
 			hbm := v2.HeartbeatMessage{Health: &healthMsg}
 			sendMessage(ws, hbm, "health")
+
+			// Record duration metric.
+			fmtScore := fmt.Sprintf("%.1f", score)
+			metrics.HealthTransmissionDuration.WithLabelValues(fmtScore).Observe(time.Since(t).Seconds())
 		}
 	}
 }

metrics/metrics.go

@@ -162,4 +162,15 @@ var (
 			Help: "Time of new registration retrieval from siteinfo.",
 		},
 	)
+
+	// HealthTransmissionDuration is a histogram for the latency of the heartbeat
+	// to assess local health and send it to the Locate.
+	HealthTransmissionDuration = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "heartbeat_health_transmission_duration",
+			Help:    "Latency for the heartbeat to assess local health and send it.",
+			Buckets: prometheus.LinearBuckets(0, 2, 30),
+		},
+		[]string{"score"},
+	)
 )

metrics/metrics_test.go

@@ -20,5 +20,6 @@ func TestLintMetrics(t *testing.T) {
 	KubernetesRequestsTotal.WithLabelValues("type", "status")
 	KubernetesRequestTimeHistogram.WithLabelValues("healthy")
 	RegistrationUpdateTime.Set(0)
+	HealthTransmissionDuration.WithLabelValues("score")
 	promtest.LintMetrics(nil)
 }