Merge branch 'metric-id'

Author: lomik

helper/point/func.go

@@ -13,7 +13,7 @@ func CleanUp(points []Point) []Point {
 	squashed := 0
 
 	for i := 0; i < l; i++ {
-		if points[i].Metric == "" || math.IsNaN(points[i].Value) {
+		if points[i].MetricID == 0 || math.IsNaN(points[i].Value) {
 			squashed++
 			continue
 		}
@@ -33,7 +33,7 @@ func Uniq(points []Point) []Point {
 	// n - position on first record with current key (metric + time)
 
 	for i = 1; i < l; i++ {
-		if points[i].Metric != points[n].Metric ||
+		if points[i].MetricID != points[n].MetricID ||
 			points[i].Time != points[n].Time {
 			n = i
 			continue
@@ -43,7 +43,7 @@ func Uniq(points []Point) []Point {
 			points[n] = points[i]
 		}
 
-		points[i].Metric = "" // mark for remove
+		points[i].MetricID = 0 // mark for remove
 	}
 
 	return CleanUp(points)
@@ -55,7 +55,7 @@ func AssertListEq(t *testing.T, expected, actual []Point) {
 	}
 
 	for i := 0; i < len(actual); i++ {
-		if (actual[i].Metric != expected[i].Metric) ||
+		if (actual[i].MetricID != expected[i].MetricID) ||
 			(actual[i].Time != expected[i].Time) ||
 			(actual[i].Timestamp != expected[i].Timestamp) ||
 			(actual[i].Value != expected[i].Value) {

helper/point/func_test.go

@@ -10,34 +10,34 @@ func TestUniq(t *testing.T) {
 	tests := [][2][]Point{
 		{
 			{ // in
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 1, Value: 1},
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 		},
 		{
 			{ // in
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 3, Value: 1},
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 3, Value: 1},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 		},
 		{
 			{ // in
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 3, Value: math.NaN()},
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: math.NaN()},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 		},
 	}
@@ -52,42 +52,42 @@ func TestCleanUp(t *testing.T) {
 	tests := [][2][]Point{
 		{
 			{ // in
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 1, Value: 1},
-				Point{Metric: "", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},
+				Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 1, Value: 1},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 		},
 		{
 			{ // in
-				Point{Metric: "", Time: 1478025152, Timestamp: 3, Value: 1},
-				Point{Metric: "", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1},
+				Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 		},
 		{
 			{ // in
-				Point{Metric: "", Time: 1478025152, Timestamp: 3, Value: 1},
-				Point{Metric: "", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "", Time: 1478025155, Timestamp: 1, Value: 1},
+				Point{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1},
+				Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 0, Time: 1478025155, Timestamp: 1, Value: 1},
 			},
 			{ // out
 			},
 		},
 		{
 			{ // in
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 3, Value: math.NaN()},
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
-				Point{Metric: "metric", Time: 1478025155, Timestamp: 1, Value: math.NaN()},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: math.NaN()},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: math.NaN()},
 			},
 			{ // out
-				Point{Metric: "metric", Time: 1478025152, Timestamp: 2, Value: 2},
+				Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},
 			},
 		},
 	}

helper/point/point.go

@@ -1,7 +1,7 @@
 package point
 
 type Point struct {
-	Metric    string
+	MetricID  uint32
 	Value     float64
 	Time      uint32
 	Timestamp uint32 // keep max if metric and time equal on two points

helper/point/points.go

@@ -0,0 +1,72 @@
+package point
+
+import "sort"
+
+type Points struct {
+	list    []Point
+	idMap   map[string]uint32
+	metrics []string
+}
+
+func NewPoints() *Points {
+	return &Points{
+		list:    make([]Point, 0),
+		idMap:   make(map[string]uint32),
+		metrics: make([]string, 0),
+	}
+}
+
+func (pp *Points) AppendPoint(metricID uint32, value float64, time uint32, version uint32) {
+	pp.list = append(pp.list, Point{
+		MetricID:  metricID,
+		Value:     value,
+		Time:      time,
+		Timestamp: version,
+	})
+}
+
+func (pp *Points) MetricID(metricName string) uint32 {
+	id := pp.idMap[metricName]
+	if id == 0 {
+		pp.metrics = append(pp.metrics, metricName)
+		id = uint32(len(pp.metrics))
+		pp.idMap[metricName] = id
+	}
+	return id
+}
+
+func (pp *Points) MetricName(metricID uint32) string {
+	i := int(metricID)
+	if i > len(pp.metrics) || i < 1 {
+		return ""
+	}
+	return pp.metrics[i-1]
+}
+
+func (pp *Points) List() []Point {
+	return pp.list
+}
+
+func (pp *Points) Len() int {
+	return len(pp.list)
+}
+
+func (pp *Points) Less(i, j int) bool {
+	if pp.list[i].MetricID == pp.list[j].MetricID {
+		return pp.list[i].Time < pp.list[j].Time
+	}
+
+	return pp.list[i].MetricID < pp.list[j].MetricID
+}
+
+func (pp *Points) Swap(i, j int) {
+	pp.list[i], pp.list[j] = pp.list[j], pp.list[i]
+}
+
+func (pp *Points) Sort() {
+	sort.Sort(pp)
+}
+
+func (pp *Points) Uniq() {
+	pp.list = Uniq(pp.list)
+}

helper/rollup/rollup.go

@@ -182,7 +182,7 @@ func doMetricPrecision(points []point.Point, precision uint32, aggr func([]point
 		points[i].Time = t
 
 		if points[n].Time == t {
-			points[i].Metric = ""
+			points[i].MetricID = 0
 		} else {
 			if i > n+1 {
 				points[n].Value = aggr(points[n:i])
@@ -199,7 +199,7 @@ func doMetricPrecision(points []point.Point, precision uint32, aggr func([]point
 
 // RollupMetric rolling up list of points of ONE metric sorted by key "time"
 // returns (new points slice, precision)
-func (r *Rollup) RollupMetric(points []point.Point) ([]point.Point, uint32) {
+func (r *Rollup) RollupMetric(metricName string, points []point.Point) ([]point.Point, uint32) {
 	// pp.Println(points)
 
 	l := len(points)
@@ -208,7 +208,7 @@ func (r *Rollup) RollupMetric(points []point.Point) ([]point.Point, uint32) {
 	}
 
 	now := uint32(time.Now().Unix())
-	rule := r.Match(points[0].Metric)
+	rule := r.Match(metricName)
 	precision := uint32(1)
 
 	for _, retention := range rule.Retention {

helper/rollup/rollup_test.go

@@ -109,13 +109,13 @@ func TestMetricPrecision(t *testing.T) {
 	tests := [][2][]point.Point{
 		{
 			{ // in
-				{Metric: "metric", Time: 1478025152, Value: 3},
-				{Metric: "metric", Time: 1478025154, Value: 2},
-				{Metric: "metric", Time: 1478025255, Value: 1},
+				{MetricID: 1, Time: 1478025152, Value: 3},
+				{MetricID: 1, Time: 1478025154, Value: 2},
+				{MetricID: 1, Time: 1478025255, Value: 1},
 			},
 			{ // out
-				{Metric: "metric", Time: 1478025120, Value: 5},
-				{Metric: "metric", Time: 1478025240, Value: 1},
+				{MetricID: 1, Time: 1478025120, Value: 5},
+				{MetricID: 1, Time: 1478025240, Value: 1},
 			},
 		},
 	}

render/data.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 	"math"
 	"strings"
@@ -54,11 +53,13 @@ func ReadUvarint(array []byte) (uint64, int, error) {
 
 type Data struct {
 	body    []byte // raw RowBinary from clickhouse
-	Points  []point.Point
+	Points  *point.Points
 	nameMap map[string]string
 	Aliases map[string][]string
 }
 
+var EmptyData *Data = &Data{Points: point.NewPoints()}
+
 func (d *Data) finalName(name string) string {
 	s, ok := d.nameMap[name]
 	if !ok {
@@ -100,24 +101,29 @@ func DataSplitFunc(data []byte, atEOF bool) (advance int, token []byte, err erro
 	return tokenLen, data[:tokenLen], nil
 }
 
-func DataParse(bodyReader io.Reader, extraPoints []point.Point, isReverse bool) (*Data, error) {
-
+func DataParse(bodyReader io.Reader, extraPoints *point.Points, isReverse bool) (*Data, error) {
 	d := &Data{
-		Points:  make([]point.Point, 0, len(extraPoints)),
-		nameMap: make(map[string]string),
+		Points: point.NewPoints(),
 	}
 
-	var p point.Point
+	pp := d.Points
 
 	// add extraPoints. With NameToID
-	for i := 0; i < len(extraPoints); i++ {
-		extraPoints[i].Metric = d.finalName(extraPoints[i].Metric)
-		d.Points = append(d.Points, extraPoints[i])
+	if extraPoints != nil {
+		extraList := extraPoints.List()
+		for i := 0; i < len(extraList); i++ {
+			pp.AppendPoint(
+				pp.MetricID(extraPoints.MetricName(extraList[i].MetricID)),
+				extraList[i].Value,
+				extraList[i].Time,
+				extraList[i].Timestamp,
+			)
+		}
 	}
 
 	nameBuf := make([]byte, 65536)
 	name := []byte{}
-	finalName := ""
+	var metricID uint32
 
 	scanner := bufio.NewScanner(bodyReader)
 	scanner.Buffer(make([]byte, 1048576), 1048576)
@@ -135,7 +141,6 @@ func DataParse(bodyReader io.Reader, extraPoints []point.Point, isReverse bool)
 		newName := row[:int(namelen)]
 		row = row[int(namelen):]
 
-		fmt.Println("cmp", string(name), string(newName))
 		if bytes.Compare(newName, name) != 0 {
 			if len(newName) > len(nameBuf) {
 				name = make([]byte, len(newName))
@@ -145,9 +150,9 @@ func DataParse(bodyReader io.Reader, extraPoints []point.Point, isReverse bool)
 				name = nameBuf[:len(newName)]
 			}
 			if isReverse {
-				finalName = d.finalName(reversePath(string(name)))
+				metricID = pp.MetricID(reversePath(string(name)))
 			} else {
-				finalName = d.finalName(string(name))
+				metricID = pp.MetricID(string(name))
 			}
 		}
 
@@ -159,11 +164,7 @@ func DataParse(bodyReader io.Reader, extraPoints []point.Point, isReverse bool)
 
 		timestamp := binary.LittleEndian.Uint32(row[:4])
 
-		p.Metric = finalName
-		p.Time = time
-		p.Value = value
-		p.Timestamp = timestamp
-		d.Points = append(d.Points, p)
+		pp.AppendPoint(metricID, value, time, timestamp)
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -172,19 +173,3 @@ func DataParse(bodyReader io.Reader, extraPoints []point.Point, isReverse bool)
 
 	return d, nil
 }
-
-func (d *Data) Len() int {
-	return len(d.Points)
-}
-
-func (d *Data) Less(i, j int) bool {
-	if d.Points[i].Metric == d.Points[j].Metric {
-		return d.Points[i].Time < d.Points[j].Time
-	}
-
-	return d.Points[i].Metric < d.Points[j].Metric
-}
-
-func (d *Data) Swap(i, j int) {
-	d.Points[i], d.Points[j] = d.Points[j], d.Points[i]
-}