Merge pull request #405 from m-lab/pollution-metrics

add metrics for pollution.

Author: yachang

metrics/metrics.go

@@ -32,6 +32,7 @@ func init() {
 	prometheus.MustRegister(PTMoreHopsAfterDest)
 	prometheus.MustRegister(PTBitsAwayFromDestV4)
 	prometheus.MustRegister(PTBitsAwayFromDestV6)
+	prometheus.MustRegister(PTPollutedCount)
 	prometheus.MustRegister(ErrorCount)
 	prometheus.MustRegister(WarningCount)
 	prometheus.MustRegister(BackendFailureCount)
@@ -266,6 +267,21 @@ var (
 		[]string{"metro"},
 	)
 
+	// Counts the PT polluted tests per metro.
+	//
+	// Provides metrics:
+	//   etl_pt_polluted_total{metro}
+	// Example usage:
+	//   metrics.PTPollutedCount.WithLabelValues("sea").Inc()
+	PTPollutedCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "etl_pt_polluted_total",
+			Help: "Count how many PT tests polluted per metro.",
+		},
+		// sea
+		[]string{"metro"},
+	)
+
 	// Counts the all warnings that do NOT result in test loss.
 	//
 	// Provides metrics:

parser/pt.go

@@ -40,6 +40,7 @@ type cachedPTData struct {
 	LogTime          time.Time
 	ConnSpec         *schema.MLabConnectionSpecification
 	LastValidHopLine string
+	MetroName        string
 }
 
 type PTParser struct {
@@ -278,7 +279,8 @@ func (pt *PTParser) ParseAndInsert(meta map[string]bigquery.Value, testName stri
 		// (in func ProcessAllNodes()). So the final parsed hop is Hops[0].
 		finalHop := PTTest.Hops[0]
 		if PTTest.ConnSpec.Client_ip != destIP && (finalHop.Dest_ip == destIP || strings.Contains(PTTest.LastValidHopLine, destIP)) {
-			// Discard pt.previousTest[index]
+			// Discard pt.previousTests[index]
+			metrics.PTPollutedCount.WithLabelValues(pt.previousTests[index].MetroName).Inc()
 			pt.previousTests = append(pt.previousTests[:index], pt.previousTests[index+1:]...)
 			break
 		}
@@ -566,5 +568,6 @@ func Parse(meta map[string]bigquery.Value, testName string, testId string, rawCo
 		LogTime:          logTime,
 		ConnSpec:         connSpec,
 		LastValidHopLine: lastValidHopLine,
+		MetroName:        metroName,
 	}, nil
 }