Revert target CPU and add more metrics (#111)

* Revert target CPU and add more metrics

* Typo

* Review comments

Author: cristinaleonr

cloudbuild/app.yaml.template

@@ -17,10 +17,8 @@ network:
     - 9090/tcp
 
 automatic_scaling:
-  max_num_instances: 20
+  max_num_instances: 40
   cool_down_period_sec: 300
-  cpu_utilization:
-    target_utilization: 0.3
 
 env_variables:
   LEGACY_SERVER: https://{{PROJECT}}.appspot.com

handler/handler.go

@@ -175,7 +175,8 @@ func (c *Client) Nearest(rw http.ResponseWriter, req *http.Request) {
 		status := http.StatusServiceUnavailable
 		result.Error = v2.NewError("nearest", "Failed to lookup nearest machines", status)
 		writeResult(rw, result.Error.Status, &result)
-		metrics.RequestsTotal.WithLabelValues("nearest", http.StatusText(result.Error.Status)).Inc()
+		metrics.RequestsTotal.WithLabelValues("nearest", "client location",
+			http.StatusText(result.Error.Status)).Inc()
 		return
 	}
 
@@ -185,7 +186,8 @@ func (c *Client) Nearest(rw http.ResponseWriter, req *http.Request) {
 	if errLat != nil || errLon != nil {
 		result.Error = v2.NewError("client", errFailedToLookupClient.Error(), http.StatusInternalServerError)
 		writeResult(rw, result.Error.Status, &result)
-		metrics.RequestsTotal.WithLabelValues("nearest", http.StatusText(result.Error.Status)).Inc()
+		metrics.RequestsTotal.WithLabelValues("nearest", "parse client location",
+			http.StatusText(result.Error.Status)).Inc()
 		return
 	}
 
@@ -197,15 +199,16 @@ func (c *Client) Nearest(rw http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		result.Error = v2.NewError("nearest", "Failed to lookup nearest machines", http.StatusInternalServerError)
 		writeResult(rw, result.Error.Status, &result)
-		metrics.RequestsTotal.WithLabelValues("nearest", http.StatusText(result.Error.Status)).Inc()
+		metrics.RequestsTotal.WithLabelValues("nearest", "server location",
+			http.StatusText(result.Error.Status)).Inc()
 		return
 	}
 
 	// Populate target URLs and write out response.
 	c.populateURLs(targets, urls, experiment, "v2", req.Form)
 	result.Results = targets
 	writeResult(rw, http.StatusOK, &result)
-	metrics.RequestsTotal.WithLabelValues("nearest", http.StatusText(http.StatusOK)).Inc()
+	metrics.RequestsTotal.WithLabelValues("nearest", "success", http.StatusText(http.StatusOK)).Inc()
 }
 
 // checkClientLocation looks up the client location and copies the location

handler/heartbeat.go

@@ -25,10 +25,11 @@ func (c *Client) Heartbeat(rw http.ResponseWriter, req *http.Request) {
 	ws, err := upgrader.Upgrade(rw, req, nil)
 	if err != nil {
 		log.Errorf("failed to establish a connection: %v", err)
-		metrics.RequestsTotal.WithLabelValues("heartbeat", err.Error()).Inc()
+		metrics.RequestsTotal.WithLabelValues("heartbeat", "establish connection",
+			"error upgrading the HTTP server connection to the WebSocket protocol").Inc()
 		return
 	}
-	metrics.RequestsTotal.WithLabelValues("heartbeat", "OK").Inc()
+	metrics.RequestsTotal.WithLabelValues("heartbeat", "establish connection", "OK").Inc()
 	go c.handleHeartbeats(ws)
 }
 

heartbeat/heartbeat.go

@@ -161,12 +161,16 @@ func (h *heartbeatStatusTracker) updatePrometheusMessage(instance v2.HeartbeatMe
 func (h *heartbeatStatusTracker) importMemorystore() {
 	values, err := h.GetAll()
 
-	if err == nil {
-		h.mu.Lock()
-		defer h.mu.Unlock()
-		h.instances = values
-		h.updateMetrics()
+	if err != nil {
+		metrics.ImportMemorystoreTotal.WithLabelValues(err.Error()).Inc()
+		return
 	}
+
+	metrics.ImportMemorystoreTotal.WithLabelValues("OK").Inc()
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.instances = values
+	h.updateMetrics()
 }
 
 // updateMetrics updates a Prometheus Gauge with the number of healthy instances per

metrics/metrics.go

@@ -16,7 +16,7 @@ var (
 			Name: "locate_requests_total",
 			Help: "Number of requests served by the Locate service.",
 		},
-		[]string{"type", "status"},
+		[]string{"type", "condition", "status"},
 	)
 
 	// AppEngineTotal counts the number of times App Engine headers are
@@ -54,6 +54,16 @@ var (
 		[]string{"experiment"},
 	)
 
+	// ImportMemorystoreTotal counts the number of times the Locate Service has imported
+	// the data in Memorystore.
+	ImportMemorystoreTotal = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "locate_import_memorystore_total",
+			Help: "Number of times the Locate Service has imported the data in Memorystore.",
+		},
+		[]string{"status"},
+	)
+
 	// PrometheusHealthCollectionDuration is a histogram that tracks the latency of the
 	// handler that collects Prometheus health signals.
 	PrometheusHealthCollectionDuration = promauto.NewHistogramVec(

metrics/metrics_test.go

@@ -7,10 +7,11 @@ import (
 )
 
 func TestLintMetrics(t *testing.T) {
-	RequestsTotal.WithLabelValues("type", "status")
+	RequestsTotal.WithLabelValues("type", "condition", "status")
 	AppEngineTotal.WithLabelValues("country")
 	CurrentHeartbeatConnections.WithLabelValues("experiment").Set(0)
 	LocateHealthStatus.WithLabelValues("experiment").Set(0)
+	ImportMemorystoreTotal.WithLabelValues("status")
 	PrometheusHealthCollectionDuration.WithLabelValues("code")
 	PortChecksTotal.WithLabelValues("status")
 	KubernetesRequestsTotal.WithLabelValues("status")