Functions: make aliasSub graphite-compatible

Previously it was operating only on metric names and that's why produced
incorrect results in some cases.

This commit changes this behavior and aliasSub now operates on full
metric name, as graphite-web do

Fixes #290

Author: Vladimir Smirnov

CHANGES.md

@@ -17,6 +17,7 @@ CHANGELOG
  - [Fix] Fix graphiteWeb proxy function (thx. to @sylvain-beugin)
  - [Fix] Prometheus Backend: correctly handle groups, fixes #405
  - [Fix] Prometheus Backend: convert target that doesn't contain seriesByTags in a same way that's used for /metrics/find
+ - [Fix] change behavior of aliasSub to match graphite-web (fixes #290)
  - [Improvement] Prometheus backend: Allow to specify "start" parameter (via backendOptions)
 **0.12.2**
  - [Fix] Fix stacked cairo-based graphs (doesn't affect json or grafana rendering)

cmd/mockbackend/aliasByTags.yaml

@@ -1,5 +1,7 @@
-pathExpression: "seriesByTag('__name__=base.metric1')"
-data:
-    - metricName: "base.metric1;foo=bar;baz=bam"
-      values: [1.0, 2.0, 3.0, 4.0, 5.0]
+expressions:
+  "seriesByTag('__name__=base.metric1')":
+    pathExpression: "seriesByTag('__name__=base.metric1')"
+    data:
+        - metricName: "base.metric1;foo=bar;baz=bam"
+          values: [1.0, 2.0, 3.0, 4.0, 5.0]
 

cmd/mockbackend/aliasSub.yaml

@@ -0,0 +1,12 @@
+expressions:
+  "dns.snake.sql_updated":
+    pathExpression: "dns.snake.sql_updated"
+    data:
+        - metricName: "dns.snake.sql_updated"
+          values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]
+  "dns.snake.zone_updated":
+    pathExpression: "dns.snake.zone_updated"
+    data:
+        - metricName: "dns.snake.zone_updated"
+          values: [2.0, .NaN, 3.0, .NaN, 5.0, 6.0]
+    

cmd/mockbackend/average.yaml

@@ -1,9 +1,11 @@
-pathExpression: "metric[123]"
-data:
-    - metricName: "metric1"
-      values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]
-    - metricName: "metric2"
-      values: [2.0, .NaN, 3.0, .NaN, 5.0, 6.0]
-    - metricName: "metric3"
-      values: [3.0, .NaN, 4.0, 5.0, 6.0, .NaN]
+expressions:
+  "metric[123]":
+    pathExpression: "metric[123]"
+    data:
+        - metricName: "metric1"
+          values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]
+        - metricName: "metric2"
+          values: [2.0, .NaN, 3.0, .NaN, 5.0, 6.0]
+        - metricName: "metric3"
+          values: [3.0, .NaN, 4.0, 5.0, 6.0, .NaN]
 

cmd/mockbackend/empty-404.yaml

@@ -0,0 +1,2 @@
+emptyBody: true
+httpCode: 404

cmd/mockbackend/empty-500.yaml

@@ -0,0 +1,2 @@
+emptyBody: true
+httpCode: 500

cmd/mockbackend/empty.yaml

@@ -1,11 +1,3 @@
 emptyBody: true
 httpCode: 200
-pathExpression: "metric[123]"
-data:
-    - metricName: "metric1"
-      values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]
-    - metricName: "metric2"
-      values: [2.0, .NaN, 3.0, .NaN, 5.0, 6.0]
-    - metricName: "metric3"
-      values: [3.0, .NaN, 4.0, 5.0, 6.0, .NaN]
 

cmd/mockbackend/i18n.yaml

@@ -1,4 +1,6 @@
-pathExpression: "test.metric.株式.rate"
-data:
-    - metricName: "test.metric.株式.rate"
-      values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]
+expressions:
+  "test.metric.株式.rate":
+    pathExpression: "test.metric.株式.rate"
+    data:
+        - metricName: "test.metric.株式.rate"
+          values: [1.0, .NaN, 2.0, 3.0, 4.0, 5.0]

cmd/mockbackend/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/go-graphite/carbonapi/zipper/httpHeaders"
 	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
+	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
 	pickle "github.com/lomik/og-rek"
 	"gopkg.in/yaml.v2"
 )
@@ -35,14 +36,64 @@ const (
 	jsonFormat responseFormat = iota
 	pickleFormat
 	protoV2Format
+	protoV3Format
 )
 
+type Metric struct {
+	MetricName string    `yaml:"metricName"`
+	Values     []float64 `yaml:"values"`
+}
+
+type Response struct {
+	PathExpression string   `yaml:"pathExpression"`
+	Data           []Metric `yaml:"data"`
+}
+
+type Config struct {
+	Code 		   int      `yaml:"httpCode"`
+	EmptyBody      bool     `yaml:"emptyBody"`
+	Expressions    map[string]Response `yaml:"expressions"`
+}
+
+func copyResponse(src Response) Response {
+	dst := Response{
+		PathExpression: src.PathExpression,
+		Data:           make([]Metric, len(src.Data)),
+	}
+
+	for i := range src.Data {
+		dst.Data[i] = Metric{
+			MetricName: src.Data[i].MetricName,
+			Values:     make([]float64, len(src.Data[i].Values)),
+		}
+
+		for j := range src.Data[i].Values {
+			dst.Data[i].Values[j] = src.Data[i].Values[j]
+		}
+	}
+
+	return dst
+}
+
+func copy(src map[string]Response) map[string]Response {
+	dst := make(map[string]Response)
+
+	for k, v := range src {
+		dst[k] = copyResponse(v)
+	}
+
+	return dst
+}
+
+var cfg = Config{}
+
 var knownFormats = map[string]responseFormat{
 	"json":            jsonFormat,
 	"pickle":          pickleFormat,
 	"protobuf":        protoV2Format,
 	"protobuf3":       protoV2Format,
 	"carbonapi_v2_pb": protoV2Format,
+	"carbonapi_v3_pb": protoV3Format,
 }
 
 func getFormat(req *http.Request) (responseFormat, error) {
@@ -64,6 +115,7 @@ func renderHandler(wr http.ResponseWriter, req *http.Request) {
 		wr.WriteHeader(cfg.Code)
 		return
 	}
+
 	format, err := getFormat(req)
 	if err != nil {
 		wr.WriteHeader(http.StatusBadRequest)
@@ -78,27 +130,61 @@ func renderHandler(wr http.ResponseWriter, req *http.Request) {
 		Metrics: []protov2.FetchResponse{},
 	}
 
-	newCfg := copy(&cfg)
+	multiv3 := protov3.MultiFetchResponse{
+		Metrics: []protov3.FetchResponse{},
+	}
 
-	for _, m := range newCfg.Data {
-		isAbsent := make([]bool, 0, len(m.Values))
-		for i := range m.Values {
-			if math.IsNaN(m.Values[i]) {
-				isAbsent = append(isAbsent, true)
-				m.Values[i] = 0.0
-			} else {
-				isAbsent = append(isAbsent, false)
-			}
+	newCfg := Config{
+		Code: cfg.Code,
+		EmptyBody: cfg.EmptyBody,
+		Expressions: copy(cfg.Expressions),
+	}
+
+	for _, target := range targets{
+		response, ok := newCfg.Expressions[target]
+		if !ok {
+			wr.WriteHeader(http.StatusNotFound)
+			wr.Write([]byte(err.Error()))
+			return
 		}
-		fr := protov2.FetchResponse{
-			Name:      m.MetricName,
-			StartTime: 1,
-			StopTime:  int32(1 + len(m.Values)),
-			StepTime:  1,
-			Values:    m.Values,
-			IsAbsent:  isAbsent,
+		for _, m := range response.Data {
+			isAbsent := make([]bool, 0, len(m.Values))
+			protov2Values := make([]float64, 0, len(m.Values))
+			for i := range m.Values {
+				if math.IsNaN(m.Values[i]) {
+					isAbsent = append(isAbsent, true)
+					protov2Values = append(protov2Values, 0.0)
+				} else {
+					isAbsent = append(isAbsent, false)
+					protov2Values = append(protov2Values, m.Values[i])
+				}
+			}
+			fr2 := protov2.FetchResponse{
+				Name:      m.MetricName,
+				StartTime: 1,
+				StopTime:  int32(1 + len(protov2Values)),
+				StepTime:  1,
+				Values:    protov2Values,
+				IsAbsent:  isAbsent,
+			}
+
+			fr3 := protov3.FetchResponse{
+				Name:                    m.MetricName,
+				PathExpression:          target,
+				ConsolidationFunc:       "avg",
+				StartTime:               1,
+				StopTime:                int64(1 + len(m.Values)),
+				StepTime:                1,
+				XFilesFactor:            0,
+				HighPrecisionTimestamps: false,
+				Values:                  m.Values,
+				RequestStartTime:        1,
+				RequestStopTime:         int64(1 + len(m.Values)),
+			}
+
+			multiv2.Metrics = append(multiv2.Metrics, fr2)
+			multiv3.Metrics = append(multiv3.Metrics, fr3)
 		}
-		multiv2.Metrics = append(multiv2.Metrics, fr)
 	}
 
 	var d []byte
@@ -111,21 +197,21 @@ func renderHandler(wr http.ResponseWriter, req *http.Request) {
 		}
 		var response []map[string]interface{}
 
-		for _, metric := range multiv2.GetMetrics() {
+		for _, metric := range multiv3.GetMetrics() {
 			var m map[string]interface{}
 
 			m = make(map[string]interface{})
 			m["start"] = metric.StartTime
 			m["step"] = metric.StepTime
 			m["end"] = metric.StopTime
 			m["name"] = metric.Name
-			m["pathExpression"] = cfg.PathExpression
+			m["pathExpression"] = metric.PathExpression
 			m["xFilesFactor"] = 0.5
 			m["consolidationFunc"] = "avg"
 
 			mv := make([]interface{}, len(metric.Values))
 			for i, p := range metric.Values {
-				if metric.IsAbsent[i] {
+				if math.IsNaN(p) {
 					mv[i] = nil
 				} else {
 					mv[i] = p
@@ -154,6 +240,18 @@ func renderHandler(wr http.ResponseWriter, req *http.Request) {
 			wr.Write([]byte(err.Error()))
 			return
 		}
+	case protoV3Format:
+		contentType = httpHeaders.ContentTypeCarbonAPIv3PB
+		if cfg.EmptyBody {
+			break
+		}
+		log.Printf("request will be served. format=protov3, data=%+v\n", multiv3)
+		d, err = multiv3.Marshal()
+		if err != nil {
+			wr.WriteHeader(http.StatusBadGateway)
+			wr.Write([]byte(err.Error()))
+			return
+		}
 	case jsonFormat:
 		contentType = "application/json"
 		if cfg.EmptyBody {
@@ -171,40 +269,6 @@ func renderHandler(wr http.ResponseWriter, req *http.Request) {
 	wr.Write(d)
 }
 
-type Metric struct {
-	MetricName string    `yaml:"metricName"`
-	Values     []float64 `yaml:"values"`
-}
-
-type Response struct {
-	Code 		   int      `yaml:"httpCode"`
-	EmptyBody      bool     `yaml:"emptyBody"`
-	PathExpression string   `yaml:"pathExpression"`
-	Data           []Metric `yaml:"data"`
-}
-
-func copy(src *Response) *Response {
-	dst := &Response{
-		PathExpression: src.PathExpression,
-		Data:           make([]Metric, len(src.Data)),
-	}
-
-	for i := range src.Data {
-		dst.Data[i] = Metric{
-			MetricName: src.Data[i].MetricName,
-			Values:     make([]float64, len(src.Data[i].Values)),
-		}
-
-		for j := range src.Data[i].Values {
-			dst.Data[i].Values[j] = src.Data[i].Values[j]
-		}
-	}
-
-	return dst
-}
-
-var cfg = Response{}
-
 func main() {
 	config := flag.String("config", "average.yaml", "yaml where it would be possible to get data")
 	address := flag.String("address", ":9070", "address to bind")
@@ -227,9 +291,10 @@ func main() {
 		return
 	}
 
-	if cfg.Code == 0 {
-		cfg.Code = http.StatusOK
-	}
+		if cfg.Code == 0 {
+			cfg.Code = http.StatusOK
+		}
+
 
 	log.Printf("started. config=%v\n", cfg)
 

cmd/mockbackend/maxSeries.yaml

@@ -0,0 +1,6 @@
+expressions:
+  "m":
+    pathExpression: "m"
+    data:
+        - metricName: "m"
+          values: [2693,1953,1642,1747,1775]