Better handling DateTime(N), where N > 3 (#1302)

Followup to this issue #1199
and this PR #1214

During testing I noticed we can't really filter for e.g. `DateTime(9)`,
because we rely there on `unixTimestampMilli`, which handles only
`DateTime(3)` precision. Fixed that here by using `toDateTime64()` which
can handle any precision.

Author: trzysiek

platform/parsers/elastic_query_dsl/dates.go

@@ -26,18 +26,18 @@ var acceptableDateTimeFormats = []string{"2006", "2006-01", "2006-01-02", "2006-
 
 // parseStrictDateOptionalTimeOrEpochMillis parses date, which is in [strict_date_optional_time || epoch_millis] format
 // (https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html)
-func (dm DateManager) parseStrictDateOptionalTimeOrEpochMillis(date any) (unixTimestamp int64, parsingSucceeded bool) {
+func (dm DateManager) parseStrictDateOptionalTimeOrEpochMillis(date any) (utcTimestamp time.Time, parsingSucceeded bool) {
 	if asInt, success := util.ExtractInt64Maybe(date); success {
-		return asInt, true
+		return time.UnixMilli(asInt), true
 	}
 
 	if asFloat, success := util.ExtractFloat64Maybe(date); success {
-		return int64(asFloat), true
+		return time.UnixMilli(int64(asFloat)), true
 	}
 
 	asString, success := date.(string)
 	if !success {
-		return -1, false
+		return time.Time{}, false
 	}
 
 	// * When missing is a single number >= 10000, it's already a unix timestamp (e.g. "10000" -> 10000th second after 01.01.1970)
@@ -55,25 +55,38 @@ func (dm DateManager) parseStrictDateOptionalTimeOrEpochMillis(date any) (unixTi
 	// It could be replaced with iso8601.ParseString() after the fixes to 1.4.0:
 	// https://github.com/relvacode/iso8601/pull/26
 	for _, format := range acceptableDateTimeFormats {
-		if date, err := time.Parse(format, asString); err == nil {
-			return date.UnixMilli(), true
+		if t, err := time.Parse(format, asString); err == nil {
+			return t, true
 		}
 	}
 
-	return -1, false
+	return time.Time{}, false
 }
 
 // ParseDateUsualFormat parses date expression, which is in [strict_date_optional_time || epoch_millis] format
 // (https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html)
 // It's most usual format for date in Kibana, used e.g. in Query DSL's range, or date_histogram.
 func (dm DateManager) ParseDateUsualFormat(exprFromRequest any, datetimeType clickhouse.DateTimeType) (
 	resultExpr model.Expr, parsingSucceeded bool) {
-	if unixTs, success := dm.parseStrictDateOptionalTimeOrEpochMillis(exprFromRequest); success {
+	if utcTs, success := dm.parseStrictDateOptionalTimeOrEpochMillis(exprFromRequest); success {
 		switch datetimeType {
 		case clickhouse.DateTime64:
-			return model.NewFunction("fromUnixTimestamp64Milli", model.NewLiteral(unixTs)), true
+			threeDigitsOfPrecisionSuffice := utcTs.UnixNano()%1_000_000 == 0
+			if threeDigitsOfPrecisionSuffice {
+				return model.NewFunction("fromUnixTimestamp64Milli", model.NewLiteral(utcTs.UnixMilli())), true
+			} else {
+				return model.NewFunction(
+					"toDateTime64",
+					model.NewInfixExpr(
+						model.NewLiteral(utcTs.UnixNano()),
+						"/",
+						model.NewLiteral(1_000_000_000),
+					),
+					model.NewLiteral(9),
+				), true
+			}
 		case clickhouse.DateTime:
-			return model.NewFunction("fromUnixTimestamp", model.NewLiteral(unixTs/1000)), true
+			return model.NewFunction("fromUnixTimestamp", model.NewLiteral(utcTs.Unix())), true
 		default:
 			logger.WarnWithCtx(dm.ctx).Msgf("Unknown datetimeType: %v", datetimeType)
 		}

platform/parsers/elastic_query_dsl/dates_test.go

@@ -8,41 +8,47 @@ import (
 	"fmt"
 	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
 func TestDateManager_parseStrictDateOptionalTimeOrEpochMillis(t *testing.T) {
+	empty := time.Time{}
 	tests := []struct {
-		missing              any
-		wantUnixTimestamp    int64
-		wantParsingSucceeded bool
+		input                  any
+		wantedTimestamp        time.Time
+		wantedParsingSucceeded bool
 	}{
-		{nil, -1, false},
-		{"2024", 1704067200000, true},
-		{int64(123), 123, true},
-		{"4234324223", 4234324223, true},
-		{"4234", 71444937600000, true},
-		{"42340", 42340, true},
-		{"42340.234", 42340, true},
-		{"2024/02", -1, false},
-		{"2024-02", 1706745600000, true},
-		{"2024-2", -1, false},
-		{"2024-02-02", 1706832000000, true},
-		{"2024-02-3", -1, false},
-		{"2024-02-30", -1, false},
-		{"2024-02-25T1", 1708822800000, true}, // this fails in Kibana, so we're better
-		{"2024-02-25T13:00:00", 1708866000000, true},
-		{"2024-02-25 13:00:00", -1, false},
-		{"2024-02-25T13:11", 1708866660000, true},
-		{"2024-02-25T25:00:00", -1, false},
-		{"2024-02-25T13:00:00+05", 1708848000000, true},
-		{"2024-02-25T13:00:00+05:00", 1708848000000, true},
+		{nil, empty, false},
+		{"2024", time.UnixMilli(1704067200000), true},
+		{int64(123), time.UnixMilli(123), true},
+		{"4234324223", time.UnixMilli(4234324223), true},
+		{"4234", time.UnixMilli(71444937600000), true},
+		{"42340", time.UnixMilli(42340), true},
+		{"42340.234", time.UnixMilli(42340), true},
+		{"2024/02", empty, false},
+		{"2024-02", time.UnixMilli(1706745600000), true},
+		{"2024-2", empty, false},
+		{"2024-02-02", time.UnixMilli(1706832000000), true},
+		{"2024-02-3", empty, false},
+		{"2024-02-30", empty, false},
+		{"2024-02-25T1", time.UnixMilli(1708822800000), true}, // this fails in Kibana, so we're better
+		{"2024-02-25T13:00:00", time.UnixMilli(1708866000000), true},
+		{"2024-02-25 13:00:00", empty, false},
+		{"2024-02-25T13:11", time.UnixMilli(1708866660000), true},
+		{"2024-02-25T25:00:00", empty, false},
+		{"2024-02-25T13:00:00+05", time.UnixMilli(1708848000000), true},
+		{"2024-02-25T13:00:00+05:00", time.UnixMilli(1708848000000), true},
+		{"2024-02-25T13:00:00.123", time.UnixMilli(1708866000123), true},
+		{"2024-02-25T13:00:00.123Z", time.UnixMilli(1708866000123), true},
+		{"2024-02-25T13:00:00.123456789", time.Unix(1708866000, 123456789), true},
+		{"2024-02-25T13:00:00.123456789Z", time.Unix(1708866000, 123456789), true},
 	}
 	for _, tt := range tests {
-		t.Run(fmt.Sprintf("%v", tt.missing), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%v", tt.input), func(t *testing.T) {
 			dm := NewDateManager(context.Background())
-			gotUnixTs, gotParsingSucceeded := dm.parseStrictDateOptionalTimeOrEpochMillis(tt.missing)
-			assert.Equalf(t, tt.wantUnixTimestamp, gotUnixTs, "MissingInDateHistogramToUnixTimestamp(%v)", tt.missing)
-			assert.Equalf(t, tt.wantParsingSucceeded, gotParsingSucceeded, "MissingInDateHistogramToUnixTimestamp(%v)", tt.missing)
+			gotUnixTs, gotParsingSucceeded := dm.parseStrictDateOptionalTimeOrEpochMillis(tt.input)
+			assert.Truef(t, tt.wantedTimestamp.Equal(gotUnixTs), "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
+			assert.Equalf(t, tt.wantedParsingSucceeded, gotParsingSucceeded, "MissingInDateHistogramToUnixTimestamp(%v)", tt.input)
 		})
 	}
 }