Skip to content

Commit 34392fa

Browse files
AleksandrMatskoAleksandrMatsko
committed
test: for building where conditions
1 parent e7f713b commit 34392fa

File tree

5 files changed

+200
-38
lines changed

5 files changed

+200
-38
lines changed

‎finder/finder.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ func newPlainFinder(ctx context.Context, config *config.Config, query string, fr
7070
if config.ClickHouse.TrySplitQuery {
7171
f = WrapSplitIndex(
7272
f,
73+
config.ClickHouse.WildcardMinDistance,
7374
config.ClickHouse.URL,
7475
config.ClickHouse.IndexTable,
7576
config.ClickHouse.IndexUseDaily,

‎finder/split.go‎

Lines changed: 45 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@ type SplitIndexFinder struct {
3232
body []byte
3333
rows [][]byte
3434
// useWrapped indicated if we should use wrapped Finder.
35-
useWrapped bool
36-
useReverse bool
35+
useWrapped bool
36+
useReverse bool
37+
wildcardMinDistance int
3738
}
3839

3940
// WrapSplitIndex wraps given finder with SplitIndexFinder logic.
4041
func WrapSplitIndex(
4142
f Finder,
43+
wildcardMinDistance int,
4244
url string,
4345
table string,
4446
dailyEnabled bool,
@@ -48,9 +50,10 @@ func WrapSplitIndex(
4850
useCache bool,
4951
) *SplitIndexFinder {
5052
return &SplitIndexFinder{
51-
wrapped: f,
52-
useWrapped: false,
53-
useReverse: false,
53+
wrapped: f,
54+
useWrapped: false,
55+
useReverse: false,
56+
wildcardMinDistance: wildcardMinDistance,
5457
indexFinderParams: indexFinderParams{
5558
url: url,
5659
table: table,
@@ -75,6 +78,8 @@ func (splitFinder *SplitIndexFinder) Execute(
7578
return errs.NewErrorWithCode("query has unmatched brackets", http.StatusBadRequest)
7679
}
7780

81+
query = where.ClearGlob(query)
82+
7883
idx := strings.IndexAny(query, "{}")
7984
if idx == -1 {
8085
splitFinder.useWrapped = true
@@ -86,13 +91,6 @@ func (splitFinder *SplitIndexFinder) Execute(
8691
return err
8792
}
8893

89-
for _, q := range splitQueries {
90-
err = validatePlainQuery(q, config.ClickHouse.WildcardMinDistance)
91-
if err != nil {
92-
return err
93-
}
94-
}
95-
9694
w, err := splitFinder.whereFilter(splitQueries, from, until)
9795
if err != nil {
9896
return err
@@ -117,8 +115,6 @@ func (splitFinder *SplitIndexFinder) Execute(
117115
}
118116

119117
func splitQuery(query string) ([]string, error) {
120-
// TODO: think about using IndexFinder.useReverse
121-
122118
splitQueries := make([]string, 0, 1)
123119

124120
firstClosingBracketIndex := strings.Index(query, "}")
@@ -138,10 +134,6 @@ func splitQuery(query string) ([]string, error) {
138134
directNodeCount := strings.Count(query[:firstOpenBracketsIndex], ".")
139135
directWildcardIndex := where.IndexWildcard(query[:firstOpenBracketsIndex])
140136
choicesInLeftMost := strings.Count(query[firstOpenBracketsIndex:firstClosingBracketIndex], ",")
141-
//fmt.Printf("\ndirect:\n\tnodeCount = %v\n\twildcardIndex = %v\n\tchoices = %v\n",
142-
// directNodeCount,
143-
// directWildcardIndex,
144-
// choicesInLeftMost)
145137

146138
lastClosingBracketIndex := strings.LastIndex(query, "}")
147139
reverseNodeCount := strings.Count(query[lastClosingBracketIndex:], ".")
@@ -152,10 +144,6 @@ func splitQuery(query string) ([]string, error) {
152144
reversWildcardIndex = where.IndexLastWildcard(query[lastClosingBracketIndex+1:])
153145
}
154146
choicesInRightMost := strings.Count(query[lastOpenBracketIndex:lastClosingBracketIndex], ",")
155-
//fmt.Printf("\nreverse:\n\tnodeCount = %v\n\twildcardIndex = %v\n\tchoices = %v\n",
156-
// reverseNodeCount,
157-
// reversWildcardIndex,
158-
// choicesInRightMost)
159147

160148
useDirect := true
161149
if directWildcardIndex >= 0 && reversWildcardIndex < 0 {
@@ -217,23 +205,48 @@ func splitPartOfQuery(prefix, queryPart, suffix string) ([]string, error) {
217205
}
218206

219207
func (splitFinder *SplitIndexFinder) whereFilter(queries []string, from, until int64) (*where.Where, error) {
220-
splitFinder.useReverse = NewIndex(
221-
splitFinder.url,
222-
splitFinder.table,
223-
splitFinder.dailyEnabled,
224-
splitFinder.reverse,
225-
splitFinder.confReverses,
226-
splitFinder.opts,
227-
splitFinder.useCache,
228-
).(*IndexFinder).useReverse(queries[0])
208+
queryWithWildcardIdx := -1
209+
for i, q := range queries {
210+
err := validatePlainQuery(q, splitFinder.wildcardMinDistance)
211+
if err != nil {
212+
return nil, err
213+
}
229214

215+
if queryWithWildcardIdx < 0 && where.HasWildcard(q) {
216+
queryWithWildcardIdx = i
217+
}
218+
}
219+
220+
if queryWithWildcardIdx >= 0 {
221+
splitFinder.useReverse = NewIndex(
222+
splitFinder.url,
223+
splitFinder.table,
224+
splitFinder.dailyEnabled,
225+
splitFinder.reverse,
226+
splitFinder.confReverses,
227+
splitFinder.opts,
228+
splitFinder.useCache,
229+
).(*IndexFinder).useReverse(queries[queryWithWildcardIdx])
230+
} else {
231+
splitFinder.useReverse = false
232+
}
233+
234+
nonWildcardQueries := make([]string, 0)
230235
aggregatedWhere := where.New()
231236
for _, q := range queries {
232237
if splitFinder.useReverse {
233238
q = ReverseString(q)
234239
}
235240

236-
aggregatedWhere.Or(where.TreeGlob("Path", q))
241+
if !where.HasWildcard(q) {
242+
nonWildcardQueries = append(nonWildcardQueries, q, q+".")
243+
} else {
244+
aggregatedWhere.Or(where.TreeGlob("Path", q))
245+
}
246+
}
247+
248+
if len(nonWildcardQueries) > 0 {
249+
aggregatedWhere.Or(where.In("Path", nonWildcardQueries))
237250
}
238251

239252
useDates := useDaily(splitFinder.dailyEnabled, from, until)

‎finder/split_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@ package finder
33
import (
44
"fmt"
55
"testing"
6+
"time"
67

8+
"github.com/lomik/graphite-clickhouse/config"
9+
"github.com/lomik/graphite-clickhouse/helper/clickhouse"
10+
"github.com/lomik/graphite-clickhouse/helper/date"
711
"github.com/stretchr/testify/assert"
812
)
913

@@ -161,3 +165,147 @@ func Test_splitQuery(t *testing.T) {
161165
})
162166
}
163167
}
168+
169+
func TestSplitIndexFinder_whereFilter(t *testing.T) {
170+
type testcase struct {
171+
name string
172+
givenQueries []string
173+
givenFrom int64
174+
givenUntil int64
175+
dailyEnabled bool
176+
wildcardMinDistance int
177+
reverse string
178+
confReverses config.IndexReverses
179+
expectedWhereStr string
180+
expectedErr error
181+
}
182+
183+
someFrom := time.Now().Unix() - 120
184+
someUntil := time.Now().Unix()
185+
186+
cases := []testcase{
187+
{
188+
name: "no wildcards in queries, no daily",
189+
givenQueries: []string{
190+
"first.metric",
191+
"second.metric",
192+
},
193+
dailyEnabled: false,
194+
expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')",
195+
},
196+
{
197+
name: "wildcard in queries, reverse preferred, no daily",
198+
givenQueries: []string{
199+
"*.first.metric",
200+
"*.second.metric",
201+
},
202+
dailyEnabled: false,
203+
expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')",
204+
},
205+
{
206+
name: "no wildcards in queries, daily enabled, but no from and until",
207+
givenQueries: []string{
208+
"first.metric",
209+
"second.metric",
210+
},
211+
dailyEnabled: true,
212+
expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')",
213+
},
214+
{
215+
name: "no wildcards in queries, daily enabled, has from, until",
216+
givenQueries: []string{
217+
"first.metric",
218+
"second.metric",
219+
},
220+
givenFrom: someFrom,
221+
givenUntil: someFrom,
222+
dailyEnabled: true,
223+
expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=2)) AND (Date >='" +
224+
date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')",
225+
},
226+
{
227+
name: "wildcard in queries, reverse preferred, daily enabled, no from, until",
228+
givenQueries: []string{
229+
"*.first.metric",
230+
"*.second.metric",
231+
},
232+
dailyEnabled: true,
233+
expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')",
234+
},
235+
{
236+
name: "wildcard in queries, reverse preferred, daily enabled, has from, until",
237+
givenQueries: []string{
238+
"*.first.metric",
239+
"*.second.metric",
240+
},
241+
dailyEnabled: true,
242+
givenFrom: someFrom,
243+
givenUntil: someUntil,
244+
expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=10003)) AND (Date >='" +
245+
date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')",
246+
},
247+
{
248+
name: "some queries have wildcard, daily enabled, has from, until",
249+
givenQueries: []string{
250+
"help.*first.metric",
251+
"help.second.metric",
252+
"help.th*rd.metric",
253+
"help.forth.metric",
254+
},
255+
dailyEnabled: true,
256+
givenFrom: someFrom,
257+
givenUntil: someUntil,
258+
expectedWhereStr: "((((Path LIKE 'help.%' AND match(Path, '^help[.]([^.]*?)first[.]metric[.]?$')) OR (Path LIKE 'help.th%' AND match(Path, '^help[.]th([^.]*?)rd[.]metric[.]?$'))) OR (Path IN ('help.second.metric','help.second.metric.','help.forth.metric','help.forth.metric.'))) AND (Level=3)) AND (Date >='" +
259+
date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')",
260+
},
261+
{
262+
name: "some queries have wildcard, daily enabled, has from, until, but reverse preferred",
263+
givenQueries: []string{
264+
"help.*first.metric.count",
265+
"help.second.metric.count",
266+
"help.th*rd.metric.count",
267+
"help.forth.metric.count",
268+
},
269+
dailyEnabled: true,
270+
givenFrom: someFrom,
271+
givenUntil: someUntil,
272+
expectedWhereStr: "((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='" +
273+
date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')",
274+
expectedErr: nil,
275+
},
276+
{
277+
name: "some queries have wildcard, daily enabled, has from, until, but reverse preferred, first query has no wildcard",
278+
givenQueries: []string{
279+
"help.second.metric.count",
280+
"help.*first.metric.count",
281+
"help.th*rd.metric.count",
282+
"help.forth.metric.count",
283+
},
284+
dailyEnabled: true,
285+
givenFrom: someFrom,
286+
givenUntil: someUntil,
287+
expectedWhereStr: "((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='" +
288+
date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')",
289+
expectedErr: nil,
290+
},
291+
}
292+
293+
for i, tc := range cases {
294+
t.Run(fmt.Sprintf("Case %v: %s", i+1, tc.name), func(t *testing.T) {
295+
f := WrapSplitIndex(
296+
&IndexFinder{},
297+
tc.wildcardMinDistance,
298+
"http://localhost:8123/",
299+
"graphite_index",
300+
tc.dailyEnabled,
301+
tc.reverse,
302+
tc.confReverses,
303+
clickhouse.Options{},
304+
false)
305+
306+
got, err := f.whereFilter(tc.givenQueries, tc.givenFrom, tc.givenUntil)
307+
assert.Equal(t, tc.expectedErr, err)
308+
assert.Equal(t, tc.expectedWhereStr, got.String())
309+
})
310+
}
311+
}

‎pkg/where/match.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ var (
99
opEq string = "="
1010
)
1111

12-
// clearGlob cleanup grafana globs like {name}
13-
func clearGlob(query string) string {
12+
// ClearGlob cleanup grafana globs like {name}
13+
func ClearGlob(query string) string {
1414
p := 0
1515
s := strings.IndexAny(query, "{[")
1616
if s == -1 {
@@ -121,7 +121,7 @@ func glob(field string, query string, optionalDotAtEnd bool) string {
121121
return ""
122122
}
123123

124-
query = clearGlob(query)
124+
query = ClearGlob(query)
125125

126126
if !HasWildcard(query) {
127127
if optionalDotAtEnd {

‎pkg/where/match_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package where
22

33
import "testing"
44

5-
func Test_clearGlob(t *testing.T) {
5+
func Test_ClearGlob(t *testing.T) {
66
type args struct {
77
query string
88
}
@@ -21,8 +21,8 @@ func Test_clearGlob(t *testing.T) {
2121
}
2222
for _, tt := range tests {
2323
t.Run(tt.query, func(t *testing.T) {
24-
if got := clearGlob(tt.query); got != tt.want {
25-
t.Errorf("clearGlob() = %v, want %v", got, tt.want)
24+
if got := ClearGlob(tt.query); got != tt.want {
25+
t.Errorf("ClearGlob() = %v, want %v", got, tt.want)
2626
}
2727
})
2828
}

0 commit comments

Comments
 (0)