Gather functions on DateTime(64) to one place

cmd/v2_test_objects.go

@@ -422,8 +422,8 @@ func (p *QueryTransformationPipeline) ComposeResult(results [][]model.QueryResul
 type QueryTransformer1 struct {
 }
 
-func (p *QueryTransformer1) Transform(queries []*model.Query) ([]*model.Query, error) {
-	logger.Debug().Msg("SimpleQueryTransformationPipeline: Transform")
+func (p *QueryTransformer1) Transform(ctx context.Context, queries []*model.Query) ([]*model.Query, error) {
+	logger.DebugWithCtx(ctx).Msg("SimpleQueryTransformationPipeline: Transform")
 	// Do basic transformation
 
 	return queries, nil

platform/clickhouse/schema.go

@@ -75,10 +75,6 @@ const (
 	Invalid
 )
 
-func (c *Column) String() string {
-	return fmt.Sprintf("%s %s", c.Name, c.Type.String())
-}
-
 func (t BaseType) String() string {
 	return t.Name
 }
@@ -326,6 +322,21 @@ func NewEmptyTable(tableName string) *Table {
 	return &Table{Name: tableName, Config: NewChTableConfigNoAttrs()}
 }
 
+func (col *Column) String() string {
+	return fmt.Sprintf("%s %s", col.Name, col.Type.String())
+}
+
+// IsDatetime <=> is it DateTime or Date (but NOT DateTime64)
+func (col *Column) IsDatetime() bool {
+	isDatetime := strings.HasPrefix(col.Type.String(), "DateTime") || strings.HasPrefix(col.Type.String(), "Date")
+	isDatetime64 := strings.HasPrefix(col.Type.String(), "DateTime64")
+	return isDatetime && !isDatetime64
+}
+
+func (col *Column) IsDatetime64() bool {
+	return strings.HasPrefix(col.Type.String(), "DateTime64")
+}
+
 func (col *Column) isArray() bool {
 	return col.Type.isArray()
 }

platform/clickhouse/util.go

@@ -3,13 +3,7 @@
 package clickhouse
 
 import (
-	"bytes"
-	"fmt"
-	"github.com/QuesmaOrg/quesma/platform/logger"
-	"github.com/QuesmaOrg/quesma/platform/model"
-	"github.com/goccy/go-json"
 	"strings"
-	"time"
 )
 
 // Code doesn't need to be pretty, 99.9% it's just for our purposes
@@ -103,85 +97,3 @@ func parseTypeFromShowColumns(typ, name string) (Type, string) {
 	}
 	return parseTypeRec(typ, name)
 }
-
-func PrettyJson(jsonStr string) string {
-	var prettyJSON bytes.Buffer
-	if err := json.Indent(&prettyJSON, []byte(jsonStr), "", "    "); err != nil {
-		return fmt.Sprintf("PrettyJson err: %v\n", err)
-	}
-	return prettyJSON.String()
-}
-
-// TimestampGroupBy returns string to be used in the select part of Clickhouse query, when grouping by timestamp interval.
-// e.g.
-// - timestampGroupBy("@timestamp", DateTime64, 30 seconds) --> toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)
-// - timestampGroupBy("@timestamp", DateTime, 30 seconds)   --> toInt64(toUnixTimestamp(`@timestamp`)/30)
-func TimestampGroupBy(timestampField model.Expr, typ DateTimeType, groupByInterval time.Duration) model.Expr {
-
-	createAExp := func(innerFuncName string, interval int64) model.Expr {
-		toUnixTsFunc := model.NewInfixExpr(
-			model.NewFunction(innerFuncName, timestampField),
-			" / ", // TODO nasty hack to make our string-based tests pass. Operator should not contain spaces obviously
-			model.NewLiteral(interval))
-		return model.NewFunction("toInt64", toUnixTsFunc)
-	}
-
-	switch typ {
-	case DateTime64:
-		// as string: fmt.Sprintf("toInt64(toUnixTimestamp(`%s`)/%f)", timestampFieldName, groupByInterval.Seconds())
-		return createAExp("toUnixTimestamp64Milli", groupByInterval.Milliseconds())
-	case DateTime:
-		return createAExp("toUnixTimestamp", groupByInterval.Milliseconds()/1000)
-	default:
-		logger.Error().Msgf("invalid timestamp fieldname: %s", timestampFieldName)
-		return model.NewLiteral("invalid") // maybe create new type InvalidExpr?
-	}
-}
-
-func TimestampGroupByWithTimezone(timestampField model.Expr, typ DateTimeType,
-	groupByInterval time.Duration, timezone string) model.Expr {
-
-	// If no timezone, or timezone is default (UTC), we just return TimestampGroupBy(...)
-	if timezone == "" {
-		return TimestampGroupBy(timestampField, typ, groupByInterval)
-	}
-
-	createAExp := func(innerFuncName string, interval, offsetMultiplier int64) model.Expr {
-		var offset model.Expr
-		offset = model.NewFunction(
-			"timeZoneOffset",
-			model.NewFunction(
-				"toTimezone",
-				timestampField, model.NewLiteral("'"+timezone+"'"),
-			),
-		)
-		if offsetMultiplier != 1 {
-			offset = model.NewInfixExpr(offset, "*", model.NewLiteral(offsetMultiplier))
-		}
-
-		unixTsWithOffset := model.NewInfixExpr(
-			model.NewFunction(innerFuncName, timestampField),
-			"+",
-			offset,
-		)
-
-		groupByExpr := model.NewInfixExpr(
-			model.NewParenExpr(unixTsWithOffset),
-			" / ", // TODO nasty hack to make our string-based tests pass. Operator should not contain spaces obviously
-			model.NewLiteral(interval),
-		)
-
-		return model.NewFunction("toInt64", groupByExpr)
-	}
-
-	switch typ {
-	case DateTime64:
-		// e.g: (toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone("timestamp",'Europe/Warsaw'))*1000) / 600000
-		return createAExp("toUnixTimestamp64Milli", groupByInterval.Milliseconds(), 1000)
-	case DateTime:
-		return createAExp("toUnixTimestamp", groupByInterval.Milliseconds()/1000, 1)
-	default:
-		logger.Error().Msgf("invalid timestamp fieldname: %s", timestampFieldName)
-		return model.NewLiteral("invalid") // maybe create new type InvalidExpr?
-	}
-}

platform/frontend_connectors/schema_search_after_transformer.go

@@ -3,6 +3,7 @@
 package frontend_connectors
 
 import (
+	"context"
 	"fmt"
 	"github.com/QuesmaOrg/quesma/platform/logger"
 	"github.com/QuesmaOrg/quesma/platform/model"
@@ -160,7 +161,7 @@ func (s searchAfterStrategyBasicAndFast) transform(query *model.Query, searchAft
 	return query, nil
 }
 
-func (s *SchemaCheckPass) applySearchAfterParameter(indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
+func (s *SchemaCheckPass) applySearchAfterParameter(ctx context.Context, indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
 	searchAfterParsed, err := s.searchAfterStrategy.validateAndParse(query, indexSchema)
 	if err != nil {
 		return nil, err

platform/frontend_connectors/schema_search_after_transformer_test.go

@@ -3,6 +3,7 @@
 package frontend_connectors
 
 import (
+	"context"
 	"fmt"
 	"github.com/QuesmaOrg/quesma/platform/clickhouse"
 	"github.com/QuesmaOrg/quesma/platform/config"
@@ -157,7 +158,7 @@ func Test_applySearchAfterParameter(t *testing.T) {
 				tc.transformedQueryExpected.SearchAfter = tc.searchAfter
 
 				transformer := NewSchemaCheckPass(&config.QuesmaConfiguration{IndexConfig: indexConfig}, tableDiscovery, strategy)
-				actual, err := transformer.applySearchAfterParameter(Schema, tc.query)
+				actual, err := transformer.applySearchAfterParameter(context.Background(), Schema, tc.query)
 				assert.Equal(t, tc.errorExpected, err != nil, "Expected error: %v, got: %v", tc.errorExpected, err)
 				if err == nil {
 					assert.Equal(t,