metrics: add route validation metrics to track processing success/failure rates (#3531)

### New Metrics Added

**Prometheus metrics:**
- `skipper_route_valid_routes` (counter) - Total number of successfully
processed routes
- `skipper_route_invalid_routes{reason}` (counter) - Total number of
invalid routes with failure reason labels

**CodaHale metrics:**
- `route.valid` (counter) - Successfully processed routes
- `route.invalid.{reason}` (counter) - Invalid routes with reason suffix

### Failure Reasons Tracked

The `reason` label/suffix captures different validation failures:
- `failed_backend_split` - Backend URL parsing errors
- `unknown_filter` - Unknown or disabled filters used
- `unknown_predicate` - Unknown predicates used
- `invalid_filter_params` - Invalid filter parameters
- `invalid_predicate_params` - Invalid predicate parameters

Signed-off-by: Veronika Volokitina <v.volokitinaa@gmail.com>

Author: vlktna

metrics/all_kind.go

@@ -121,6 +121,16 @@ func (a *All) IncErrorsStreaming(routeId string) {
 
 }
 
+func (a *All) IncValidRoutes() {
+	a.prometheus.IncValidRoutes()
+	a.codaHale.IncValidRoutes()
+}
+
+func (a *All) IncInvalidRoutes(reason string) {
+	a.prometheus.IncInvalidRoutes(reason)
+	a.codaHale.IncInvalidRoutes(reason)
+}
+
 func (a *All) Close() {
 	a.codaHale.Close()
 	a.prometheus.Close()

metrics/codahale.go

@@ -33,6 +33,8 @@ const (
 
 	KeyErrorsBackend   = "errors.backend.%s"
 	KeyErrorsStreaming = "errors.streaming.%s"
+	KeyValidRoutes     = "route.valid"
+	KeyInvalidRoutes   = "route.invalid.%s"
 
 	statsRefreshDuration = time.Duration(5 * time.Second)
 
@@ -256,6 +258,14 @@ func (c *CodaHale) IncErrorsStreaming(routeId string) {
 	}
 }
 
+func (c *CodaHale) IncValidRoutes() {
+	c.incCounter(KeyValidRoutes, 1)
+}
+
+func (c *CodaHale) IncInvalidRoutes(reason string) {
+	c.incCounter(fmt.Sprintf(KeyInvalidRoutes, reason), 1)
+}
+
 func (c *CodaHale) Close() {
 	close(c.quit)
 }

metrics/metrics.go

@@ -78,6 +78,8 @@ type Metrics interface {
 	IncErrorsStreaming(routeId string)
 	RegisterHandler(path string, handler *http.ServeMux)
 	UpdateGauge(key string, value float64)
+	IncValidRoutes()
+	IncInvalidRoutes(reason string)
 	Close()
 }
 

metrics/metricstest/metricsmock.go

@@ -202,4 +202,12 @@ func (m *MockMetrics) Gauge(key string) (v float64, ok bool) {
 	return
 }
 
+func (m *MockMetrics) IncValidRoutes() {
+	m.IncCounter("route.valid")
+}
+
+func (m *MockMetrics) IncInvalidRoutes(reason string) {
+	m.IncCounter("route.invalid." + reason)
+}
+
 func (m *MockMetrics) Close() {}

metrics/prometheus.go

@@ -53,6 +53,8 @@ type Prometheus struct {
 	customHistogramM           *prometheus.HistogramVec
 	customCounterM             *prometheus.CounterVec
 	customGaugeM               *prometheus.GaugeVec
+	validRoutesM               *prometheus.CounterVec
+	invalidRoutesM             *prometheus.CounterVec
 
 	opts     Options
 	registry *prometheus.Registry
@@ -271,6 +273,20 @@ func NewPrometheus(opts Options) *Prometheus {
 		Buckets:   opts.HistogramBuckets,
 	}, []string{"key"}))
 
+	p.validRoutesM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: promRouteSubsystem,
+		Name:      "valid_routes",
+		Help:      "Total number of successfully processed routes.",
+	}, []string{}))
+
+	p.invalidRoutesM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: promRouteSubsystem,
+		Name:      "invalid_routes",
+		Help:      "Total number of invalid routes with failure reasons.",
+	}, []string{"reason"}))
+
 	// Register prometheus runtime collectors if required.
 	if opts.EnableRuntimeMetrics {
 		register(p, collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
@@ -472,6 +488,16 @@ func (p *Prometheus) IncErrorsStreaming(routeID string) {
 	p.proxyStreamingErrorsM.WithLabelValues(routeID).Inc()
 }
 
+// IncValidRoutes satisfies Metrics interface.
+func (p *Prometheus) IncValidRoutes() {
+	p.validRoutesM.WithLabelValues().Inc()
+}
+
+// IncInvalidRoutes satisfies Metrics interface.
+func (p *Prometheus) IncInvalidRoutes(reason string) {
+	p.invalidRoutesM.WithLabelValues(reason).Inc()
+}
+
 func (p *Prometheus) Close() {}
 
 // withStartLabelGatherer adds a "start" label to all counters with

routing/datasource.go

@@ -23,6 +23,19 @@ const (
 
 var errInvalidWeightParams = errors.New("invalid argument for the Weight predicate")
 
+type invalidDefinitionError string
+
+func (e invalidDefinitionError) Error() string { return string(e) }
+func (e invalidDefinitionError) Code() string  { return string(e) }
+
+var (
+	errUnknownFilter          = invalidDefinitionError("unknown_filter")
+	errInvalidFilterParams    = invalidDefinitionError("invalid_filter_params")
+	errUnknownPredicate       = invalidDefinitionError("unknown_predicate")
+	errInvalidPredicateParams = invalidDefinitionError("invalid_predicate_params")
+	errFailedBackendSplit     = invalidDefinitionError("failed_backend_split")
+)
+
 func (it incomingType) String() string {
 	switch it {
 	case incomingReset:
@@ -219,26 +232,26 @@ func createFilter(o *Options, def *eskip.Filter, cpm map[string]PredicateSpec) (
 	spec, ok := o.FilterRegistry[def.Name]
 	if !ok {
 		if isTreePredicate(def.Name) || def.Name == predicates.HostName || def.Name == predicates.PathRegexpName || def.Name == predicates.MethodName || def.Name == predicates.HeaderName || def.Name == predicates.HeaderRegexpName {
-			return nil, fmt.Errorf("trying to use %q as filter, but it is only available as predicate", def.Name)
+			return nil, fmt.Errorf("%w: trying to use %q as filter, but it is only available as predicate", errUnknownFilter, def.Name)
 		}
 
 		if _, ok := cpm[def.Name]; ok {
-			return nil, fmt.Errorf("trying to use %q as filter, but it is only available as predicate", def.Name)
+			return nil, fmt.Errorf("%w: trying to use %q as filter, but it is only available as predicate", errUnknownFilter, def.Name)
 		}
 
-		return nil, fmt.Errorf("filter %q not found", def.Name)
+		return nil, fmt.Errorf("%w: filter %q not found", errUnknownFilter, def.Name)
 	}
 
 	start := time.Now()
 
 	f, err := spec.CreateFilter(def.Args)
 
-	if o.Metrics != nil { // measure regardless of the error
+	if o.Metrics != nil {
 		o.Metrics.MeasureFilterCreate(def.Name, start)
 	}
 
 	if err != nil {
-		return nil, fmt.Errorf("failed to create filter %q: %w", spec.Name(), err)
+		return nil, fmt.Errorf("%w: failed to create filter %q: %w", errInvalidFilterParams, spec.Name(), err)
 	}
 	return f, nil
 }
@@ -370,7 +383,7 @@ func parseWeightPredicateArgs(args []interface{}) (int, error) {
 }
 
 // initialize predicate instances from their spec with the concrete arguments
-func processPredicates(cpm map[string]PredicateSpec, defs []*eskip.Predicate) ([]Predicate, int, error) {
+func processPredicates(o *Options, cpm map[string]PredicateSpec, defs []*eskip.Predicate) ([]Predicate, int, error) {
 	cps := make([]Predicate, 0, len(defs))
 	var weight int
 	for _, def := range defs {
@@ -379,7 +392,7 @@ func processPredicates(cpm map[string]PredicateSpec, defs []*eskip.Predicate) ([
 			var err error
 
 			if w, err = parseWeightPredicateArgs(def.Args); err != nil {
-				return nil, 0, err
+				return nil, 0, fmt.Errorf("%w: %w", errInvalidPredicateParams, err)
 			}
 
 			weight += w
@@ -393,12 +406,12 @@ func processPredicates(cpm map[string]PredicateSpec, defs []*eskip.Predicate) ([
 
 		spec, ok := cpm[def.Name]
 		if !ok {
-			return nil, 0, fmt.Errorf("predicate %q not found", def.Name)
+			return nil, 0, fmt.Errorf("%w: predicate %q not found", errUnknownPredicate, def.Name)
 		}
 
 		cp, err := spec.Create(def.Args)
 		if err != nil {
-			return nil, 0, fmt.Errorf("failed to create predicate %q: %w", spec.Name(), err)
+			return nil, 0, fmt.Errorf("%w: failed to create predicate %q: %w", errInvalidPredicateParams, spec.Name(), err)
 		}
 
 		if ws, ok := spec.(WeightedPredicateSpec); ok {
@@ -473,7 +486,7 @@ func processTreePredicates(r *Route, predicateList []*eskip.Predicate) error {
 func processRouteDef(o *Options, cpm map[string]PredicateSpec, def *eskip.Route) (*Route, error) {
 	scheme, host, err := splitBackend(def)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: %w", errFailedBackendSplit, err)
 	}
 
 	fs, err := createFilters(o, def.Filters, cpm)
@@ -486,7 +499,7 @@ func processRouteDef(o *Options, cpm map[string]PredicateSpec, def *eskip.Route)
 		return nil, err
 	}
 
-	cps, weight, err := processPredicates(cpm, def.Predicates)
+	cps, weight, err := processPredicates(o, cpm, def.Predicates)
 	if err != nil {
 		return nil, err
 	}
@@ -496,6 +509,10 @@ func processRouteDef(o *Options, cpm map[string]PredicateSpec, def *eskip.Route)
 		return nil, err
 	}
 
+	if o.Metrics != nil {
+		o.Metrics.IncValidRoutes()
+	}
+
 	return r, nil
 }
 
@@ -519,6 +536,13 @@ func processRouteDefs(o *Options, defs []*eskip.Route) (routes []*Route, invalid
 		} else {
 			invalidDefs = append(invalidDefs, def)
 			o.Log.Errorf("failed to process route %s: %v", def.Id, err)
+
+			var defErr invalidDefinitionError
+			if errors.As(err, &defErr) && o.Metrics != nil {
+				o.Metrics.IncInvalidRoutes(defErr.Code())
+			} else if o.Metrics != nil {
+				o.Metrics.IncInvalidRoutes("other")
+			}
 		}
 	}
 	return