Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 5b2cf47

Browse files
trzysiektrzysiek
authored
Accept new DateTime format (with timezone) in ids query (#1489)
Adds missing feature from #1488 Before - we only accepted either dates with no timezone, or trickily removed `+00:00 UTC` (thus only to accept this timezone). Other timezones resulted in an error and our error response. Now - we properly parse the entire date with timezone, and it works e.g. for the user's `+0200 CEST`. I reproduced the error, after this PR it no longer appears (tested `ids` query manually, which is the only 1 where this error occured, and it properly returned my record from Clickhouse)
1 parent 45484a1 commit 5b2cf47

File tree

4 files changed

+149
-17
lines changed

4 files changed

+149
-17
lines changed

‎platform/parsers/elastic_query_dsl/dates.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@ func NewDateManager(ctx context.Context) DateManager {
2222
}
2323

2424
var acceptableDateTimeFormats = []string{"2006", "2006-01", "2006-01-02", "2006-01-02", "2006-01-02T15",
25-
"2006-01-02T15:04", "2006-01-02T15:04:05", "2006-01-02T15:04:05Z07", "2006-01-02T15:04:05Z07:00"}
25+
"2006-01-02T15:04", "2006-01-02T15:04:05", "2006-01-02T15:04:05Z07", "2006-01-02T15:04:05Z07:00",
26+
"2006-01-02 15:04:05 -0700 MST", "2006-01-02 15:04:05.000 -0700 MST", "2006-01-02 15:04:05.000000000 -0700 MST",
27+
"2006-01-02 15:04:05", "2006-01-02 15:04:05.000", "2006-01-02 15:04:05.000000000",
28+
// All those from the line above theorically shouldn't be accepted, but it can't hurt to also accept them.
29+
// It also simplifies parsing of `ids` query.
30+
}
2631

2732
// parseStrictDateOptionalTimeOrEpochMillis parses date, which is in [strict_date_optional_time || epoch_millis] format
2833
// (https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html)

‎platform/parsers/elastic_query_dsl/dates_test.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func TestDateManager_parseStrictDateOptionalTimeOrEpochMillis(t *testing.T) {
3434
{"2024-02-30", empty, false},
3535
{"2024-02-25T1", time.UnixMilli(1708822800000), true}, // this fails in Kibana, so we're better
3636
{"2024-02-25T13:00:00", time.UnixMilli(1708866000000), true},
37-
{"2024-02-25 13:00:00", empty, false},
37+
{"2024-02-25 13:00:00", time.UnixMilli(1708866000000), true},
3838
{"2024-02-25T13:11", time.UnixMilli(1708866660000), true},
3939
{"2024-02-25T25:00:00", empty, false},
4040
{"2024-02-25T13:00:00+05", time.UnixMilli(1708848000000), true},
@@ -43,12 +43,19 @@ func TestDateManager_parseStrictDateOptionalTimeOrEpochMillis(t *testing.T) {
4343
{"2024-02-25T13:00:00.123Z", time.UnixMilli(1708866000123), true},
4444
{"2024-02-25T13:00:00.123456789", time.Unix(1708866000, 123456789), true},
4545
{"2024-02-25T13:00:00.123456789Z", time.Unix(1708866000, 123456789), true},
46+
{"2024-12-21 07:29:03.367 +0000 UTC", time.UnixMilli(1734766143367), true},
47+
{"2025-06-16 12:59:59 +0200 CEST", time.Unix(1750071599, 0), true},
48+
{"2025-06-16 12:59:59.985 +0200 CEST", time.UnixMilli(1750071599985), true},
49+
{"2025-06-16 12:59:59.985233345 +0200 CEST", time.Unix(1750071599, 985233345), true},
4650
}
4751
for i, tt := range tests {
4852
t.Run(util.PrettyTestName(fmt.Sprintf("%v", tt.input), i), func(t *testing.T) {
4953
dm := NewDateManager(context.Background())
5054
gotUnixTs, gotParsingSucceeded := dm.parseStrictDateOptionalTimeOrEpochMillis(tt.input)
51-
assert.Truef(t, tt.wantedTimestamp.Equal(gotUnixTs), "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
55+
assert.Truef(t, tt.wantedTimestamp.Equal(gotUnixTs),
56+
"MissingInDateHistogramToUnixTimestamp(\n input: %v,\n wanted: %v,\n got: %v\n gotUnix: %v,\n"+
57+
" gotUnixMilli: %v,\n gotUnixNano: %v\n)", tt.input, tt.wantedTimestamp,
58+
gotUnixTs, gotUnixTs.Unix(), gotUnixTs.UnixMilli(), gotUnixTs.UnixNano())
5259
assert.Equalf(t, tt.wantedParsingSucceeded, gotParsingSucceeded, "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
5360
})
5461
}

‎platform/parsers/elastic_query_dsl/query_parser.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -340,11 +340,25 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
340340
for i, id := range ids {
341341
idInHex := strings.Split(id, uuidSeparator)[0]
342342
if idAsStr, err := hex.DecodeString(idInHex); err != nil {
343-
logger.Error().Msgf("error parsing document id %s: %v", id, err)
343+
logger.ErrorWithCtx(cw.Ctx).Msgf("error parsing document id %s: %v", id, err)
344344
return model.NewSimpleQueryInvalid()
345345
} else {
346-
tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
347-
ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
346+
// Before:
347+
// tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
348+
// ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
349+
//
350+
// Now we stop trimming, instead parse the date ourselves, and then output in UTC.
351+
dm := NewDateManager(cw.Ctx)
352+
353+
if tsAsTime, ok := dm.parseStrictDateOptionalTimeOrEpochMillis(string(idAsStr)); ok {
354+
tsUTC := tsAsTime.UTC()
355+
tsGoodFormat := tsUTC.Format("2006-01-02 15:04:05.000000000")
356+
tsTrimmedNano := strings.TrimRight(tsGoodFormat, "0")
357+
ids[i] = fmt.Sprintf("'%s'", tsTrimmedNano)
358+
} else {
359+
logger.ErrorWithCtx(cw.Ctx).Msgf("error parsing document id %s:, idAsStr: %v", id, idAsStr)
360+
return model.NewSimpleQueryInvalid()
361+
}
348362
}
349363
uniqueIds = append(uniqueIds, id)
350364
}

‎platform/testdata/requests.go‎

Lines changed: 117 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2402,6 +2402,111 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
24022402
[]string{},
24032403
},
24042404
{ // [44]
2405+
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 2.
2406+
"ids, 1 value, different DateTime format: with timezone, precision: 2",
2407+
`{
2408+
"query": {
2409+
"ids": {
2410+
"values": ["323032352d30372d30362030393a33383a30332e3132202b3030303020555443qqq3635363236333330333833373335333633363331333736333335333736353632363333313337333233313330363536353330333236313332363633323631333833323634333833313333333033333636333933353332333033363631333533363335333833323333363233393333333733333635333733353338333736333632"]
2411+
}
2412+
},
2413+
"track_total_hits": false
2414+
}`,
2415+
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:38:03.12',2)`},
2416+
model.ListAllFields,
2417+
[]string{
2418+
`SELECT "message" ` +
2419+
`FROM ` + TableName + ` ` +
2420+
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:38:03.12',2) ` +
2421+
`LIMIT 10`,
2422+
},
2423+
[]string{},
2424+
},
2425+
{ // [45]
2426+
// important test, DateTime64(3) is pretty standard
2427+
"ids, 1 value, different DateTime format: with timezone, precision: 3",
2428+
`{
2429+
"query": {
2430+
"ids": {
2431+
"values": ["323032352d30372d30342031353a33323a34332e333737202b303230302043455354qqq3332363233363331363636353633333933323338363133353339333233323330333036313335333833343332363536333633363533343330363333373632363333393636363233303337333936313632333136323330333736313633333933303635333436313336363133383632333433313330363133353634333733353631"]
2432+
}
2433+
},
2434+
"track_total_hits": false
2435+
}`,
2436+
[]string{`"@timestamp" = toDateTime64('2025-07-04 13:32:43.377',3)`},
2437+
model.ListAllFields,
2438+
[]string{
2439+
`SELECT "message" ` +
2440+
`FROM ` + TableName + ` ` +
2441+
`WHERE "@timestamp" = toDateTime64('2025-07-04 13:32:43.377',3) ` +
2442+
`LIMIT 10`,
2443+
},
2444+
[]string{},
2445+
},
2446+
{ // [46]
2447+
// important test, DateTime64(9) is pretty standard
2448+
"ids, 1 value, different DateTime format: with timezone, precision: 9",
2449+
`{
2450+
"query": {
2451+
"ids": {
2452+
"values": ["323032352d30372d30362031303a31313a30332e313233343536373839202b3030303020555443qqq3338363633373635363433333334333333353331333936333334363336333634333836313632363136343634333633343634363433393337333333393338333933323634333533393334333936333635363333353338333233313331363436313337333533333338333133333339333933383335333033393636363633343636"]
2453+
}
2454+
},
2455+
"track_total_hits": false
2456+
}`,
2457+
[]string{`"@timestamp" = toDateTime64('2025-07-06 10:11:03.123456789',9)`},
2458+
model.ListAllFields,
2459+
[]string{
2460+
`SELECT "message" ` +
2461+
`FROM ` + TableName + ` ` +
2462+
`WHERE "@timestamp" = toDateTime64('2025-07-06 10:11:03.123456789',9) ` +
2463+
`LIMIT 10`,
2464+
},
2465+
[]string{},
2466+
},
2467+
{ // [47]
2468+
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 7.
2469+
"ids, 1 value, different DateTime format: with timezone, precision: 7",
2470+
`{
2471+
"query": {
2472+
"ids": {
2473+
"values": ["323032352d30372d30362030393a33363a30332e32353531323336202b3030303020555443qqq3338333636363634333733363634333533363333333336363339333736343330363136323334363136343631363533333336363133313636333236323337333936313632333133323335333733363632363633313335333136323334333336333636333833373333363333343331363336313330333133363636333136353631"]
2474+
}
2475+
},
2476+
"track_total_hits": false
2477+
}`,
2478+
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:36:03.2551236',7)`},
2479+
model.ListAllFields,
2480+
[]string{
2481+
`SELECT "message" ` +
2482+
`FROM ` + TableName + ` ` +
2483+
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:36:03.2551236',7) ` +
2484+
`LIMIT 10`,
2485+
},
2486+
[]string{},
2487+
},
2488+
{ // [48]
2489+
// DateTime64(3 or 9) are "normal"/standard. We check weird one like 7.
2490+
"ids, 1 value, different DateTime format: with timezone, precision: 7, but timestamp with only 1 (.1)",
2491+
`{
2492+
"query": {
2493+
"ids": {
2494+
"values": ["323032352d30372d30362030393a33383a30332e31202b3030303020555443qqq3339333533343339333033303332333533363631333033323333333936363335333636333339363436363632333336323336363233383332333233353335363233343631363436363332363433383331363636333636333033353333333833363631333533333338333133303334333336313634333733393631363333333633"]
2495+
}
2496+
},
2497+
"track_total_hits": false
2498+
}`,
2499+
[]string{`"@timestamp" = toDateTime64('2025-07-06 09:38:03.1',1)`},
2500+
model.ListAllFields,
2501+
[]string{
2502+
`SELECT "message" ` +
2503+
`FROM ` + TableName + ` ` +
2504+
`WHERE "@timestamp" = toDateTime64('2025-07-06 09:38:03.1',1) ` +
2505+
`LIMIT 10`,
2506+
},
2507+
[]string{},
2508+
},
2509+
{ // [49]
24052510
"ids, 2+ values",
24062511
`{
24072512
"query": {
@@ -2424,7 +2529,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
24242529
},
24252530
[]string{},
24262531
},
2427-
{ // [45]
2532+
{ // [50]
24282533
"ids with DateTime64(9) (trailing zeroes)",
24292534
`{
24302535
"query": {
@@ -2434,17 +2539,17 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
24342539
},
24352540
"track_total_hits": false
24362541
}`,
2437-
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.367000000',9)`},
2542+
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.367',3)`},
24382543
model.ListAllFields,
24392544
[]string{
24402545
`SELECT "message" ` +
24412546
`FROM ` + TableName + ` ` +
2442-
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.367000000',9) ` +
2547+
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.367',3) ` +
24432548
`LIMIT 10000`,
24442549
},
24452550
[]string{},
24462551
},
2447-
{ // [46]
2552+
{ // [51]
24482553
"ids with DateTime64(9) (no trailing zeroes)",
24492554
`{
24502555
"query": {
@@ -2464,7 +2569,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
24642569
},
24652570
[]string{},
24662571
},
2467-
{ // [47]
2572+
{ // [52]
24682573
"ids with DateTime64(0)",
24692574
`{
24702575
"query": {
@@ -2474,17 +2579,18 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
24742579
},
24752580
"track_total_hits": false
24762581
}`,
2477-
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03',0)`},
2582+
[]string{`"@timestamp" = toDateTime64('2024-12-21 07:29:03.',0)`},
2583+
// dot at the end doesn't matter - CH accepts it exactly like it wasn't there
24782584
model.ListAllFields,
24792585
[]string{
24802586
`SELECT "message" ` +
24812587
`FROM ` + TableName + ` ` +
2482-
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03',0) ` +
2588+
`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.',0) ` +
24832589
`LIMIT 10000`,
24842590
},
24852591
[]string{},
24862592
},
2487-
{ // [48]
2593+
{ // [53]
24882594
"ids with DateTime64(1)",
24892595
`{
24902596
"query": {
@@ -2504,7 +2610,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
25042610
},
25052611
[]string{},
25062612
},
2507-
{ // [49]
2613+
{ // [54]
25082614
Name: "range with int as datetime. when all query tests use transformers, expected results should be different",
25092615
QueryJson: `
25102616
{
@@ -2534,7 +2640,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
25342640
`LIMIT 10`,
25352641
},
25362642
},
2537-
{ // [50]
2643+
{ // [55]
25382644
Name: "range with int not as datetime. when all query tests use transformers, expected results should be different",
25392645
QueryJson: `
25402646
{
@@ -2563,7 +2669,7 @@ Men\\'s Clothing \\\\ %' LIMIT 10`},
25632669
`LIMIT 10`,
25642670
},
25652671
},
2566-
{ // [51]
2672+
{ // [56]
25672673
"_index term",
25682674
`{
25692675
"query": { /*one comment */

0 commit comments

Comments
 (0)