Merge branch 'master' into feat/add_linters

Author: kissken

autocomplete/autocomplete.go

@@ -529,8 +529,9 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 			valueSQL = fmt.Sprintf("substr(Tag1, %d) AS value", len(tag)+2)
 			wr.And(where.HasPrefix("Tag1", tag+"="+valuePrefix))
 		} else {
-			valueSQL = fmt.Sprintf("substr(arrayJoin(Tags), %d) AS value", len(tag)+2)
-			wr.And(where.HasPrefix("arrayJoin(Tags)", tag+"="+valuePrefix))
+			prefixSelector := where.HasPrefix("x", tag+"="+valuePrefix)
+			valueSQL = fmt.Sprintf("substr(arrayFilter(x -> %s, Tags)[1], %d) AS value", prefixSelector, len(tag)+2)
+			wr.And("arrayExists(x -> " + prefixSelector + ", Tags)")
 		}
 
 		wr.Andf("Date >= '%s' AND Date <= '%s'", fromDate, untilDate)

autocomplete/autocomplete_test.go

@@ -68,7 +68,7 @@ func TestHandler_ServeValues(t *testing.T) {
 	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
 
 	srv.AddResponce(
-		"SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+
+		"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
 			"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
 		&chtest.TestResponse{
 			Body: []byte("host1\nhost2\ndc-host2\ndc-host3\n"),
@@ -129,7 +129,7 @@ func TestTagsAutocomplete_ServeValuesCached(t *testing.T) {
 	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
 
 	srv.AddResponce(
-		"SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+
+		"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
 			"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
 		&chtest.TestResponse{
 			Body: []byte("host1\nhost2\ndc-host2\ndc-host3\n"),

config/config.go

@@ -235,6 +235,7 @@ type ClickHouse struct {
 	IndexReverses        IndexReverses         `toml:"index-reverses"           json:"index-reverses"           comment:"see doc/config.md"                                                                                                              commented:"true"`
 	IndexTimeout         time.Duration         `toml:"index-timeout"            json:"index-timeout"            comment:"total timeout to fetch series list from index"`
 	TaggedTable          string                `toml:"tagged-table"             json:"tagged-table"             comment:"'tagged' table from carbon-clickhouse, required for seriesByTag"`
+	TagsCountTable       string                `toml:"tags-count-table"         json:"tags-count-table"         comment:"Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md"`
 	TaggedAutocompleDays int                   `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days" comment:"or how long the daemon will query tags during autocomplete"`
 	TaggedUseDaily       bool                  `toml:"tagged-use-daily"         json:"tagged-use-daily"         comment:"whether to use date filter when searching for the metrics in the tagged-table"`
 	TaggedCosts          map[string]*Costs     `toml:"tagged-costs"             json:"tagged-costs"             comment:"costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags" commented:"true"`

deploy/doc/config.md

@@ -199,4 +199,30 @@ When `reverse = true` is set for data-table, there are two possibles cases for [
 Depends on it for having a proper retention and aggregation you must additionally set `rollup-use-reverted = true` for the first case and `rollup-use-reverted = false` for the second.
 
 #### Additional tuning tagged find for seriesByTag and autocomplete
-Only one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/
+Only one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/
+
+To always choose the best Tag1 you can set the parameter `tag1-count-table = <table_name>`. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this:
+
+```
+CREATE TABLE IF NOT EXISTS default.tag1_count_per_day
+(
+  Date Date,
+  Tag1 String,
+  Count UInt64
+)
+ENGINE = SummingMergeTree
+ORDER BY (Date, Tag1);
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS
+SELECT Date AS Date,
+       Tag1 AS Tag1,
+       count(*) AS Count
+FROM default.graphite_tags
+GROUP BY (Date, Tag1);
+```
+
+Here we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written.
+graphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table.
+Overall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases.
+
+Note that this option only works for terms with '=' operator in them.

doc/config.md

@@ -203,6 +203,33 @@ Depends on it for having a proper retention and aggregation you must additionall
 
 #### Additional tuning tagged find for seriesByTag and autocomplete
 Only one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/
+
+To always choose the best Tag1 you can set the parameter `tag1-count-table = <table_name>`. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this:
+
+```
+CREATE TABLE IF NOT EXISTS default.tag1_count_per_day
+(
+  Date Date,
+  Tag1 String,
+  Count UInt64
+)
+ENGINE = SummingMergeTree
+ORDER BY (Date, Tag1);
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS
+SELECT Date AS Date,
+       Tag1 AS Tag1,
+       count(*) AS Count
+FROM default.graphite_tags
+GROUP BY (Date, Tag1);
+```
+
+Here we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written.
+graphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table.
+Overall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases.
+
+Note that this option only works for terms with '=' operator in them.
+
 ```toml
 [common]
  # general listener
@@ -353,6 +380,8 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
  index-timeout = "1m0s"
  # 'tagged' table from carbon-clickhouse, required for seriesByTag
  tagged-table = "graphite_tagged"
+ # Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md
+ tags-count-table = ""
  # or how long the daemon will query tags during autocomplete
  tagged-autocomplete-days = 7
  # whether to use date filter when searching for the metrics in the tagged-table

finder/finder.go

@@ -41,6 +41,7 @@ func newPlainFinder(ctx context.Context, config *config.Config, query string, fr
 		f = NewTagged(
 			config.ClickHouse.URL,
 			config.ClickHouse.TaggedTable,
+			config.ClickHouse.TagsCountTable,
 			config.ClickHouse.TaggedUseDaily,
 			config.FeatureFlags.UseCarbonBehavior,
 			config.FeatureFlags.DontMatchMissingTags,
@@ -147,6 +148,7 @@ func FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm,
 	fnd := NewTagged(
 		config.ClickHouse.URL,
 		config.ClickHouse.TaggedTable,
+		config.ClickHouse.TagsCountTable,
 		config.ClickHouse.TaggedUseDaily,
 		config.FeatureFlags.UseCarbonBehavior,
 		config.FeatureFlags.DontMatchMissingTags,