Fix *seriesLists functions (e.x. divideSeriesLists) to work properly with unsorted metrics

 * Sort denominators and numerators
 * Add test config for mockbackend to replicate this scenario

Author: Vladimir Smirnov

CHANGES.md

@@ -25,6 +25,7 @@ CHANGELOG
  - [Fix] Honor SendGlobAsIs and AlwaysSendGlobAsIs (important for pre-0.12 configs)
  - [Fix] Fix panic in some cases when one of the metrics is missing
  - [Fix] Compatbility fixes to useAboveSeries (now it's behavior matches graphite-web's)
+ - [Fix] Fix seriesList functions in case of unsorted responses (\*seriesList sorts denomniators and numerators first of all)
  - [Improvement] Redesign error handling and logging. Logging should be now less noisy and all error messages should contain better reasoning about error cause
  - [Improvement] Move more of the logging messages to Debug level - that should make logs less noisy and still preserve ability to see detailed errors on Debug level
  - [Improvement] Add a config parameter to disable tldCache (useful for clickhouse type backends)

cmd/mockbackend/main.go

@@ -5,17 +5,19 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/ansel1/merry"
-	"github.com/go-graphite/carbonapi/intervalset"
-	"go.uber.org/zap"
 	"io/ioutil"
 	"log"
 	"math"
+	"math/rand"
 	"net/http"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/ansel1/merry"
+	"github.com/go-graphite/carbonapi/intervalset"
+	"go.uber.org/zap"
+
 	"github.com/go-graphite/carbonapi/zipper/httpHeaders"
 	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
 	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
@@ -78,10 +80,11 @@ type MultiListenerConfig struct {
 }
 
 type Config struct {
-	Address     string              `yaml:"address"`
-	Code        int                 `yaml:"httpCode"`
-	EmptyBody   bool                `yaml:"emptyBody"`
-	Expressions map[string]Response `yaml:"expressions"`
+	Address        string              `yaml:"address"`
+	Code           int                 `yaml:"httpCode"`
+	ShuffleResults bool                `yaml:"shuffleResults"`
+	EmptyBody      bool                `yaml:"emptyBody"`
+	Expressions    map[string]Response `yaml:"expressions"`
 }
 
 func copyResponse(src Response) Response {
@@ -243,6 +246,12 @@ func (cfg *listener) findHandler(wr http.ResponseWriter, req *http.Request) {
 			})
 	}
 
+	if cfg.Config.ShuffleResults {
+		rand.Shuffle(len(multiGlobs.Metrics), func(i, j int) {
+			multiGlobs.Metrics[i], multiGlobs.Metrics[j] = multiGlobs.Metrics[j], multiGlobs.Metrics[i]
+		})
+	}
+
 	logger.Info("will return", zap.Any("response", multiGlobs))
 
 	var b []byte
@@ -397,6 +406,15 @@ func (cfg *listener) renderHandler(wr http.ResponseWriter, req *http.Request) {
 		}
 	}
 
+	if cfg.Config.ShuffleResults {
+		rand.Shuffle(len(multiv2.Metrics), func(i, j int) {
+			multiv2.Metrics[i], multiv2.Metrics[j] = multiv2.Metrics[j], multiv2.Metrics[i]
+		})
+		rand.Shuffle(len(multiv3.Metrics), func(i, j int) {
+			multiv3.Metrics[i], multiv3.Metrics[j] = multiv3.Metrics[j], multiv3.Metrics[i]
+		})
+	}
+
 	var d []byte
 	var contentType string
 	switch format {

cmd/mockbackend/sortedResponses.yaml

@@ -0,0 +1,35 @@
+# Example:
+# wget -q -O- "http://localhost:8081/render/?format=json&target=divideSeriesLists(seriesByTag('name=~exec_vgcache.cache_read_hits','subtype=dp1','vg_name=dp1_sdai'),seriesByTag('name=~exec_vgcache.cache_read.*','subtype=dp1','vg_name=dp1_sdai'))"   | jq -r '.[] | .target'
+listeners:
+        - shuffleResults: true
+          expressions:
+            "seriesByTag('name=~exec_vgcache.cache_read_hits','subtype=dp1','vg_name=dp1_sdai')":
+              pathExpression: "seriesByTag('name=~exec_vgcache.cache_read_hits','subtype=dp1','vg_name=dp1_dai')"
+              data:
+                - metricName: "exec_vgcache.cache_read_hits;host=dp1-osd1;vg_name=dp1_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp1-osd1;vg_name=dp1_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp2-osd1;vg_name=dp2_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp2-osd1;vg_name=dp2_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp3-osd1;vg_name=dp3_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp3-osd1;vg_name=dp3_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]
+            "seriesByTag('name=~exec_vgcache.cache_read.*','subtype=dp1','vg_name=dp1_sdai')":
+              pathExpression: "seriesByTag('name=~exec_vgcache.cache_read.*','subtype=dp1','vg_name=dp1_sdai')"
+              data:
+                - metricName: "exec_vgcache.cache_read_hits;host=dp1-osd1;vg_name=dp1_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp1-osd1;vg_name=dp1_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp2-osd1;vg_name=dp2_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp2-osd1;vg_name=dp2_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp3-osd1;vg_name=dp3_sdai;avg=true"
+                  values: [2.0, 2.0, 2.0, 2.0]
+                - metricName: "exec_vgcache.cache_read_hits;host=dp3-osd1;vg_name=dp3_sdai;sum=true"
+                  values: [4.0, 4.0, 4.0, 4.0]

expr/functions/seriesList/function.go

@@ -3,6 +3,7 @@ package seriesList
 import (
 	"fmt"
 	"math"
+	"sort"
 	"strconv"
 
 	"github.com/go-graphite/carbonapi/expr/helper"
@@ -55,6 +56,7 @@ func (f *seriesList) Do(e parser.Expr, from, until int64, values map[parser.Metr
 			return nil, nil
 		}
 	}
+	sort.Slice(numerators, func(i, j int) bool { return numerators[i].Name < numerators[j].Name })
 
 	denominators, err := helper.GetSeriesArg(e.Args()[1], from, until, values)
 	if err != nil {
@@ -71,6 +73,7 @@ func (f *seriesList) Do(e parser.Expr, from, until int64, values map[parser.Metr
 			return nil, nil
 		}
 	}
+	sort.Slice(numerators, func(i, j int) bool { return denominators[i].Name < denominators[j].Name })
 
 	sizeMatch := len(denominators) == len(numerators) || len(denominators) == 1
 	useMatching, err := e.GetBoolNamedOrPosArgDefault("matching", 2, !useConstant && !sizeMatch)