Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit dc8c289

Browse files
jakozaurjakozaur
authored
Support timestamp in iso 8601 format (#210)
Why: - our prospect got many failed queries into that trap - Elastic and Opensearch support [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) - Clickhouse also support it as part of https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#parsedatetimebesteffort - Go native support of ISO 8601 is poor, but there is good library for that
1 parent e0fc128 commit dc8c289

File tree

8 files changed

+148
-86
lines changed

8 files changed

+148
-86
lines changed

‎quesma/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ require (
3232
github.com/mitchellh/copystructure v1.2.0 // indirect
3333
github.com/mitchellh/reflectwalk v1.0.2 // indirect
3434
github.com/pkg/errors v0.9.1 // indirect
35+
github.com/relvacode/iso8601 v1.4.0 // indirect
3536
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a // indirect
3637
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
3738
)

‎quesma/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
103103
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
104104
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
105105
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
106+
github.com/relvacode/iso8601 v1.4.0 h1:GsInVSEJfkYuirYFxa80nMLbH2aydgZpIf52gYZXUJs=
107+
github.com/relvacode/iso8601 v1.4.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
106108
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
107109
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
108110
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=

‎quesma/queryparser/query_parser.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@ package queryparser
22

33
import (
44
"encoding/json"
5-
wc "mitmproxy/quesma/queryparser/where_clause"
6-
"mitmproxy/quesma/quesma/types"
7-
85
"fmt"
96
"github.com/k0kubun/pp"
7+
"github.com/relvacode/iso8601"
108
"mitmproxy/quesma/clickhouse"
119
"mitmproxy/quesma/logger"
1210
"mitmproxy/quesma/model"
1311
"mitmproxy/quesma/queryparser/lucene"
1412
"mitmproxy/quesma/queryparser/query_util"
13+
wc "mitmproxy/quesma/queryparser/where_clause"
14+
"mitmproxy/quesma/quesma/types"
15+
"mitmproxy/quesma/util"
1516
"strconv"
1617
"strings"
17-
"time"
1818
"unicode"
1919
)
2020

@@ -777,7 +777,9 @@ func (cw *ClickhouseQueryTranslator) parseRange(queryMap QueryMap) model.SimpleQ
777777
isDatetimeInDefaultFormat = false
778778
}
779779

780-
for op, v := range v.(QueryMap) {
780+
keysSorted := util.MapKeysSorted(v.(QueryMap))
781+
for _, op := range keysSorted {
782+
v := v.(QueryMap)[op]
781783
var fieldToPrint, timeFormatFuncName string
782784
var valueToCompare wc.Statement
783785
fieldType := cw.Table.GetDateTimeType(cw.Ctx, field)
@@ -793,7 +795,7 @@ func (cw *ClickhouseQueryTranslator) parseRange(queryMap QueryMap) model.SimpleQ
793795
if dateTime, ok := v.(string); ok {
794796
// if it's a date, we need to parse it to Clickhouse's DateTime format
795797
// how to check if it does not contain date math expression?
796-
if _, err := time.Parse(time.RFC3339Nano, dateTime); err == nil {
798+
if _, err := iso8601.ParseString(dateTime); err == nil {
797799
vToPrint, timeFormatFuncName = cw.parseDateTimeString(cw.Table, field, dateTime)
798800
// TODO Investigate the quotation below
799801
valueToCompare = wc.NewFunction(timeFormatFuncName, wc.NewLiteral(fmt.Sprintf("'%s'", dateTime)))

‎quesma/queryparser/query_parser_range_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package queryparser
2+
3+
import (
4+
"context"
5+
"github.com/stretchr/testify/assert"
6+
"mitmproxy/quesma/clickhouse"
7+
"mitmproxy/quesma/concurrent"
8+
"mitmproxy/quesma/quesma/config"
9+
"testing"
10+
)
11+
12+
type parseRangeTest struct {
13+
name string
14+
rangePartOfQuery QueryMap
15+
createTableQuery string
16+
expectedWhere string
17+
}
18+
19+
var parseRangeTests = []parseRangeTest{
20+
{
21+
"DateTime64",
22+
QueryMap{
23+
"timestamp": QueryMap{
24+
"format": "strict_date_optional_time",
25+
"gte": "2024-02-02T13:47:16.029Z",
26+
"lte": "2024-02-09T13:47:16.029Z",
27+
},
28+
},
29+
`CREATE TABLE ` + tableName + `
30+
( "message" String, "timestamp" DateTime64(3, 'UTC') )
31+
ENGINE = Memory`,
32+
`"timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z')`,
33+
},
34+
{
35+
"parseDateTimeBestEffort",
36+
QueryMap{
37+
"timestamp": QueryMap{
38+
"format": "strict_date_optional_time",
39+
"gte": "2024-02-02T13:47:16.029Z",
40+
"lte": "2024-02-09T13:47:16.029Z",
41+
},
42+
},
43+
`CREATE TABLE ` + tableName + `
44+
( "message" String, "timestamp" DateTime )
45+
ENGINE = Memory`,
46+
`"timestamp">=parseDateTimeBestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTimeBestEffort('2024-02-09T13:47:16.029Z')`,
47+
},
48+
{
49+
"numeric range",
50+
QueryMap{
51+
"time_taken": QueryMap{
52+
"gt": "100",
53+
},
54+
},
55+
`CREATE TABLE ` + tableName + `
56+
( "message" String, "timestamp" DateTime, "time_taken" UInt32 )
57+
ENGINE = Memory`,
58+
`"time_taken">100`,
59+
},
60+
{
61+
"DateTime64",
62+
QueryMap{
63+
"timestamp": QueryMap{
64+
"format": "strict_date_optional_time",
65+
"gte": "2024-02-02T13:47:16",
66+
"lte": "2024-02-09T13:47:16",
67+
},
68+
},
69+
`CREATE TABLE ` + tableName + `
70+
( "message" String, "timestamp" DateTime64(3, 'UTC') )
71+
ENGINE = Memory`,
72+
`"timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16')`,
73+
},
74+
}
75+
76+
func Test_parseRange(t *testing.T) {
77+
for _, test := range parseRangeTests {
78+
t.Run(test.name, func(t *testing.T) {
79+
table, err := clickhouse.NewTable(test.createTableQuery, clickhouse.NewNoTimestampOnlyStringAttrCHConfig())
80+
if err != nil {
81+
t.Fatal(err)
82+
}
83+
assert.NoError(t, err)
84+
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, table), config.QuesmaConfiguration{})
85+
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
86+
87+
whereClause := cw.parseRange(test.rangePartOfQuery).Sql.Stmt
88+
assert.Equal(t, test.expectedWhere, whereClause)
89+
})
90+
}
91+
}

‎quesma/queryparser/query_parser_test.go‎

Lines changed: 6 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,14 @@ func TestQueryParserStringAttrConfig(t *testing.T) {
5454
t.Run(tt.Name, func(t *testing.T) {
5555
simpleQuery, queryInfo, _, _ := cw.ParseQueryInternal(tt.QueryJson)
5656
assert.True(t, simpleQuery.CanParse, "can parse")
57+
simpleQuery, queryInfo, _, _ = cw.ParseQueryInternal(tt.QueryJson)
5758
assert.Contains(t, tt.WantedSql, simpleQuery.Sql.Stmt, "contains wanted sql")
5859
assert.Equal(t, tt.WantedQueryType, queryInfo.Typ, "equals to wanted query type")
59-
query := cw.BuildNRowsQuery("*", simpleQuery, model.DefaultSizeListQuery)
60+
size := model.DefaultSizeListQuery
61+
if queryInfo.Size != 0 {
62+
size = queryInfo.Size
63+
}
64+
query := cw.BuildNRowsQuery("*", simpleQuery, size)
6065
assert.Contains(t, tt.WantedQuery, *query)
6166
// Test the new WhereStatement
6267
if simpleQuery.Sql.WhereStatement != nil {
@@ -336,77 +341,6 @@ func TestNew(t *testing.T) {
336341
}
337342
}
338343

339-
// Test_parseRange tests if DateTime64 field properly uses Clickhouse's 'parseDateTime64BestEffort' function
340-
func Test_parseRange_DateTime64(t *testing.T) {
341-
rangePartOfQuery := QueryMap{
342-
"timestamp": QueryMap{
343-
"format": "strict_date_optional_time",
344-
"gte": "2024-02-02T13:47:16.029Z",
345-
"lte": "2024-02-09T13:47:16.029Z",
346-
},
347-
}
348-
table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+`
349-
( "message" String, "timestamp" DateTime64(3, 'UTC') )
350-
ENGINE = Memory`,
351-
clickhouse.NewNoTimestampOnlyStringAttrCHConfig(),
352-
)
353-
if err != nil {
354-
t.Fatal(err)
355-
}
356-
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, table), config.QuesmaConfiguration{})
357-
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
358-
359-
whereClause := cw.parseRange(rangePartOfQuery).Sql.Stmt
360-
split := strings.Split(whereClause, "parseDateTime64BestEffort")
361-
assert.Len(t, split, 3)
362-
}
363-
364-
// Test_parseRange tests if DateTime field properly uses Clickhouse's 'parseDateTimeBestEffort' function
365-
func Test_parseRange_DateTime(t *testing.T) {
366-
rangePartOfQuery := QueryMap{
367-
"timestamp": QueryMap{
368-
"format": "strict_date_optional_time",
369-
"gte": "2024-02-02T13:47:16.029Z",
370-
"lte": "2024-02-09T13:47:16.029Z",
371-
},
372-
}
373-
table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+`
374-
( "message" String, "timestamp" DateTime )
375-
ENGINE = Memory`,
376-
clickhouse.NewNoTimestampOnlyStringAttrCHConfig(),
377-
)
378-
if err != nil {
379-
t.Fatal(err)
380-
}
381-
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, table), config.QuesmaConfiguration{})
382-
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
383-
384-
whereClause := cw.parseRange(rangePartOfQuery).Sql.Stmt
385-
split := strings.Split(whereClause, "parseDateTimeBestEffort")
386-
assert.Len(t, split, 3)
387-
}
388-
389-
func Test_parseRange_numeric(t *testing.T) {
390-
rangePartOfQuery := QueryMap{
391-
"time_taken": QueryMap{
392-
"gt": "100",
393-
},
394-
}
395-
table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+`
396-
( "message" String, "timestamp" DateTime, "time_taken" UInt32 )
397-
ENGINE = Memory`,
398-
clickhouse.NewNoTimestampOnlyStringAttrCHConfig(),
399-
)
400-
if err != nil {
401-
t.Fatal(err)
402-
}
403-
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, table), config.QuesmaConfiguration{})
404-
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
405-
406-
whereClause := cw.parseRange(rangePartOfQuery).Sql.Stmt
407-
assert.Equal(t, "\"time_taken\">100", whereClause)
408-
}
409-
410344
func Test_parseSortFields(t *testing.T) {
411345
tests := []struct {
412346
name string

‎quesma/quesma/search_test.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,6 @@ func TestAsyncSearchHandler(t *testing.T) {
7474
for i, tt := range testdata.TestsAsyncSearch {
7575
t.Run(strconv.Itoa(i)+tt.Name, func(t *testing.T) {
7676
db, mock, err := sqlmock.New()
77-
if tt.Name == "Histogram: possible query nr 2" {
78-
queryMatcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error {
79-
fmt.Printf("JM SQL: %s\n", actualSQL)
80-
return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL)
81-
})
82-
db, mock, err = sqlmock.New(sqlmock.QueryMatcherOption(queryMatcher))
83-
}
8477
if err != nil {
8578
t.Fatal(err)
8679
}
@@ -172,6 +165,7 @@ func TestSearchHandler(t *testing.T) {
172165
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
173166
for _, tt := range testdata.TestsSearch {
174167
t.Run(tt.Name, func(t *testing.T) {
168+
175169
db, mock, err := sqlmock.New()
176170
if err != nil {
177171
t.Fatal(err)

‎quesma/testdata/requests.go‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1781,7 +1781,7 @@ var TestsSearch = []SearchTestCase{
17811781
}`,
17821782
[]string{""},
17831783
model.ListByField,
1784-
[]model.Query{newSimplestQuery()},
1784+
[]model.Query{withLimit(newSimplestQuery(), 500)},
17851785
[]string{`SELECT "message" FROM "logs-generic-default" LIMIT 500`},
17861786
},
17871787
{ // [26]
@@ -1952,6 +1952,39 @@ var TestsSearch = []SearchTestCase{
19521952
[]model.Query{justSimplestWhere(`"user.id"='kimchy'`)},
19531953
[]string{qToStr(justSimplestWhere(`"user.id"='kimchy'`))},
19541954
},
1955+
{ // [35] terms with range
1956+
"Terms with range",
1957+
`{
1958+
"size": 1,
1959+
"query": {
1960+
"bool": {
1961+
"filter": [
1962+
{
1963+
"terms": {
1964+
"cliIP": [
1965+
"2601:204:c503:c240:9c41:5531:ad94:4d90",
1966+
"50.116.43.98",
1967+
"75.246.0.64"
1968+
]
1969+
}
1970+
},
1971+
{
1972+
"range": {
1973+
"@timestamp": {
1974+
"gte": "2024-05-16T00:00:00",
1975+
"lte": "2024-05-17T23:59:59"
1976+
}
1977+
}
1978+
}
1979+
]
1980+
}
1981+
}
1982+
}`,
1983+
[]string{`("cliIP"='2601:204:c503:c240:9c41:5531:ad94:4d90' OR "cliIP"='50.116.43.98' OR "cliIP"='75.246.0.64') AND ("@timestamp">=parseDateTime64BestEffort('2024-05-16T00:00:00') AND "@timestamp"<=parseDateTime64BestEffort('2024-05-17T23:59:59'))`},
1984+
model.Normal,
1985+
[]model.Query{withLimit(justSimplestWhere(`("cliIP"='2601:204:c503:c240:9c41:5531:ad94:4d90' OR "cliIP"='50.116.43.98' OR "cliIP"='75.246.0.64') AND ("@timestamp">=parseDateTime64BestEffort('2024-05-16T00:00:00') AND "@timestamp"<=parseDateTime64BestEffort('2024-05-17T23:59:59'))`), 1)},
1986+
[]string{qToStr(withLimit(justSimplestWhere(`("cliIP"='2601:204:c503:c240:9c41:5531:ad94:4d90' OR "cliIP"='50.116.43.98' OR "cliIP"='75.246.0.64') AND ("@timestamp">=parseDateTime64BestEffort('2024-05-16T00:00:00') AND "@timestamp"<=parseDateTime64BestEffort('2024-05-17T23:59:59'))`), 1))},
1987+
},
19551988
}
19561989

19571990
var TestsSearchNoAttrs = []SearchTestCase{

‎quesma/testdata/util.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@ func justSimplestWhere(whereClause string) model.Query {
6464
return query
6565
}
6666

67+
func withLimit(query model.Query, limit int) model.Query {
68+
query.SuffixClauses = []string{"LIMIT " + strconv.Itoa(limit)}
69+
return query
70+
}
71+
6772
// EscapeBrackets is a simple helper function used in sqlmock's tests.
6873
// Example usage: sqlmock.ExpectQuery(EscapeBrackets(`SELECT count() FROM "logs-generic-default" WHERE `))
6974
func EscapeBrackets(s string) string {

0 commit comments

Comments
 (0)