Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Accept new DateTime format (with timezone) in ids query #1489

Merged
merged 6 commits into from
Jul 7, 2025
Merged

Accept new DateTime format (with timezone) in ids query #1489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
024b0cf
parsing new timestamp format works
trzysiek Jul 5, 2025
fe981e9
fix debug error + add test copied from the user's Issue
trzysiek Jul 5, 2025
a1a5412
All test pass (including new one). It should work now, need to test m…
trzysiek Jul 5, 2025
61be4cd
remove debug test
trzysiek Jul 6, 2025
5481085
add a few more tests (+ tested them manually)
trzysiek Jul 6, 2025
b733344
remove debug
trzysiek Jul 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion platform/parsers/elastic_query_dsl/dates.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ func NewDateManager(ctx context.Context) DateManager {
}

var acceptableDateTimeFormats = []string{"2006", "2006-01", "2006-01-02", "2006-01-02", "2006-01-02T15",
"2006-01-02T15:04", "2006-01-02T15:04:05", "2006-01-02T15:04:05Z07", "2006-01-02T15:04:05Z07:00"}
"2006-01-02T15:04", "2006-01-02T15:04:05", "2006-01-02T15:04:05Z07", "2006-01-02T15:04:05Z07:00",
"2006-01-02 15:04:05 -0700 MST", "2006-01-02 15:04:05.000 -0700 MST", "2006-01-02 15:04:05.000000000 -0700 MST",
"2006-01-02 15:04:05", "2006-01-02 15:04:05.000", "2006-01-02 15:04:05.000000000",
// All those from the line above theorically shouldn't be accepted, but it can't hurt to also accept them.
// It also simplifies parsing of `ids` query.
}

// parseStrictDateOptionalTimeOrEpochMillis parses date, which is in [strict_date_optional_time || epoch_millis] format
// (https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html)
Expand Down
11 changes: 9 additions & 2 deletions platform/parsers/elastic_query_dsl/dates_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestDateManager_parseStrictDateOptionalTimeOrEpochMillis(t *testing.T) {
{"2024-02-30", empty, false},
{"2024-02-25T1", time.UnixMilli(1708822800000), true}, // this fails in Kibana, so we're better
{"2024-02-25T13:00:00", time.UnixMilli(1708866000000), true},
{"2024-02-25 13:00:00", empty, false},
{"2024-02-25 13:00:00", time.UnixMilli(1708866000000), true},
{"2024-02-25T13:11", time.UnixMilli(1708866660000), true},
{"2024-02-25T25:00:00", empty, false},
{"2024-02-25T13:00:00+05", time.UnixMilli(1708848000000), true},
Expand All @@ -43,12 +43,19 @@ func TestDateManager_parseStrictDateOptionalTimeOrEpochMillis(t *testing.T) {
{"2024-02-25T13:00:00.123Z", time.UnixMilli(1708866000123), true},
{"2024-02-25T13:00:00.123456789", time.Unix(1708866000, 123456789), true},
{"2024-02-25T13:00:00.123456789Z", time.Unix(1708866000, 123456789), true},
{"2024-12-21 07:29:03.367 +0000 UTC", time.UnixMilli(1734766143367), true},
{"2025-06-16 12:59:59 +0200 CEST", time.Unix(1750071599, 0), true},
{"2025-06-16 12:59:59.985 +0200 CEST", time.UnixMilli(1750071599985), true},
{"2025-06-16 12:59:59.985233345 +0200 CEST", time.Unix(1750071599, 985233345), true},
}
for i, tt := range tests {
t.Run(util.PrettyTestName(fmt.Sprintf("%v", tt.input), i), func(t *testing.T) {
dm := NewDateManager(context.Background())
gotUnixTs, gotParsingSucceeded := dm.parseStrictDateOptionalTimeOrEpochMillis(tt.input)
assert.Truef(t, tt.wantedTimestamp.Equal(gotUnixTs), "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
assert.Truef(t, tt.wantedTimestamp.Equal(gotUnixTs),
"MissingInDateHistogramToUnixTimestamp(\n input: %v,\n wanted: %v,\n got: %v\n gotUnix: %v,\n"+
" gotUnixMilli: %v,\n gotUnixNano: %v\n)", tt.input, tt.wantedTimestamp,
gotUnixTs, gotUnixTs.Unix(), gotUnixTs.UnixMilli(), gotUnixTs.UnixNano())
assert.Equalf(t, tt.wantedParsingSucceeded, gotParsingSucceeded, "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
})
}
Expand Down
20 changes: 17 additions & 3 deletions platform/parsers/elastic_query_dsl/query_parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,11 +340,25 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
for i, id := range ids {
idInHex := strings.Split(id, uuidSeparator)[0]
if idAsStr, err := hex.DecodeString(idInHex); err != nil {
logger.Error().Msgf("error parsing document id %s: %v", id, err)
logger.ErrorWithCtx(cw.Ctx).Msgf("error parsing document id %s: %v", id, err)
return model.NewSimpleQueryInvalid()
} else {
tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
// Before:
// tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
// ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
//
// Now we stop trimming, instead parse the date ourselves, and then output in UTC.
dm := NewDateManager(cw.Ctx)

if tsAsTime, ok := dm.parseStrictDateOptionalTimeOrEpochMillis(string(idAsStr)); ok {
tsUTC := tsAsTime.UTC()
tsGoodFormat := tsUTC.Format("2006-01-02 15:04:05.000000000")
tsTrimmedNano := strings.TrimRight(tsGoodFormat, "0")
ids[i] = fmt.Sprintf("'%s'", tsTrimmedNano)
} else {
logger.ErrorWithCtx(cw.Ctx).Msgf("error parsing document id %s:, idAsStr: %v", id, idAsStr)
return model.NewSimpleQueryInvalid()
}
}
uniqueIds = append(uniqueIds, id)
}
Expand Down
128 changes: 117 additions & 11 deletions platform/testdata/requests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2402,6 +2402,111 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
[]string{},
},
{ // [44]
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 2.
"ids, 1 value, different DateTime format: with timezone, precision: 2",
`{
"query": {
"ids": {
"values": ["323032352d30372d30362030393a33383a30332e3132202b3030303020555443qqq3635363236333330333833373335333633363331333736333335333736353632363333313337333233313330363536353330333236313332363633323631333833323634333833313333333033333636333933353332333033363631333533363335333833323333363233393333333733333635333733353338333736333632"]
}
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:38:03.12',2)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:38:03.12',2) ` +
`LIMIT 10`,
},
[]string{},
},
{ // [45]
// important test, DateTime64(3) is pretty standard
"ids, 1 value, different DateTime format: with timezone, precision: 3",
`{
"query": {
"ids": {
"values": ["323032352d30372d30342031353a33323a34332e333737202b303230302043455354qqq3332363233363331363636353633333933323338363133353339333233323330333036313335333833343332363536333633363533343330363333373632363333393636363233303337333936313632333136323330333736313633333933303635333436313336363133383632333433313330363133353634333733353631"]
}
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2025-07-04 13:32:43.377',3)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2025-07-04 13:32:43.377',3) ` +
`LIMIT 10`,
},
[]string{},
},
{ // [46]
// important test, DateTime64(9) is pretty standard
"ids, 1 value, different DateTime format: with timezone, precision: 9",
`{
"query": {
"ids": {
"values": ["323032352d30372d30362031303a31313a30332e313233343536373839202b3030303020555443qqq3338363633373635363433333334333333353331333936333334363336333634333836313632363136343634333633343634363433393337333333393338333933323634333533393334333936333635363333353338333233313331363436313337333533333338333133333339333933383335333033393636363633343636"]
}
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2025-07-06 10:11:03.123456789',9)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2025-07-06 10:11:03.123456789',9) ` +
`LIMIT 10`,
},
[]string{},
},
{ // [47]
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 7.
"ids, 1 value, different DateTime format: with timezone, precision: 7",
`{
"query": {
"ids": {
"values": ["323032352d30372d30362030393a33363a30332e32353531323336202b3030303020555443qqq3338333636363634333733363634333533363333333336363339333736343330363136323334363136343631363533333336363133313636333236323337333936313632333133323335333733363632363633313335333136323334333336333636333833373333363333343331363336313330333133363636333136353631"]
}
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:36:03.2551236',7)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:36:03.2551236',7) ` +
`LIMIT 10`,
},
[]string{},
},
{ // [48]
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 7.
"ids, 1 value, different DateTime format: with timezone, precision: 7, but timestamp with only 1 (.1)",
`{
"query": {
"ids": {
"values": ["323032352d30372d30362030393a33383a30332e31202b3030303020555443qqq3339333533343339333033303332333533363631333033323333333936363335333636333339363436363632333336323336363233383332333233353335363233343631363436363332363433383331363636333636333033353333333833363631333533333338333133303334333336313634333733393631363333333633"]
}
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:38:03.1',1)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:38:03.1',1) ` +
`LIMIT 10`,
},
[]string{},
},
{ // [49]
"ids, 2+ values",
`{
"query": {
Expand All @@ -2424,7 +2529,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
},
[]string{},
},
{ // [45]
{ // [50]
"ids with DateTime64(9) (trailing zeroes)",
`{
"query": {
Expand All @@ -2434,17 +2539,17 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.367000000',9)`},
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.367',3)`},
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.367000000',9) ` +
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.367',3) ` +
`LIMIT 10000`,
},
[]string{},
},
{ // [46]
{ // [51]
"ids with DateTime64(9) (no trailing zeroes)",
`{
"query": {
Expand All @@ -2464,7 +2569,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
},
[]string{},
},
{ // [47]
{ // [52]
"ids with DateTime64(0)",
`{
"query": {
Expand All @@ -2474,17 +2579,18 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
},
"track_total_hits": false
}`,
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03',0)`},
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.',0)`},
// dot at the end doesn't matter - CH accepts it exactly like it wasn't there
model.ListAllFields,
[]string{
`SELECT "message" ` +
`FROM ` + TableName + ` ` +
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03',0) ` +
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.',0) ` +
`LIMIT 10000`,
},
[]string{},
},
{ // [48]
{ // [53]
"ids with DateTime64(1)",
`{
"query": {
Expand All @@ -2504,7 +2610,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
},
[]string{},
},
{ // [49]
{ // [54]
Name: "range with int as datetime. when all query tests use transformers, expected results should be different",
QueryJson: `
{
Expand Down Expand Up @@ -2534,7 +2640,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
`LIMIT 10`,
},
},
{ // [50]
{ // [55]
Name: "range with int not as datetime. when all query tests use transformers, expected results should be different",
QueryJson: `
{
Expand Down Expand Up @@ -2563,7 +2669,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
`LIMIT 10`,
},
},
{ // [51]
{ // [56]
"_index term",
`{
"query": { /*one comment */
Expand Down
Loading