Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
fe31d19
Most done
trzysiek Jan 6, 2025
4b3bc1f
Merge branch 'main' into datetime
trzysiek Jan 6, 2025
1cdf1f9
ALMOST works. Finish quesma/TestHandlingDateTimeFields.
trzysiek Jan 6, 2025
4913873
Should work, but doesn't (bad AsString)
trzysiek Jan 10, 2025
bca80d3
Done
trzysiek Jan 11, 2025
c919e13
All tests pass
trzysiek Jan 12, 2025
05715d7
Merge branch 'main' into datetime
trzysiek Jan 12, 2025
4841458
Cleanup
trzysiek Jan 12, 2025
df5e80c
Merge branch 'main' into datetime
trzysiek Feb 9, 2025
39359e4
After merge quickfixes
trzysiek Feb 9, 2025
3a946db
Before final refactor
trzysiek Feb 9, 2025
37e9241
close to finish
trzysiek Feb 9, 2025
b8df985
99.9% done
trzysiek Feb 10, 2025
81441b8
Done?
trzysiek Feb 10, 2025
0f37df0
Fix license
trzysiek Feb 10, 2025
54bdae5
Fix staticcheck
trzysiek Feb 10, 2025
a701c91
Cleanup
trzysiek Feb 10, 2025
4991bd7
Merge branch 'main' into datetime
trzysiek Feb 10, 2025
f40c067
Fix linter
trzysiek Feb 10, 2025
34cb438
Cleanup
trzysiek Feb 10, 2025
b141c80
Cleanup
trzysiek Feb 10, 2025
d89a4a9
Unskip test
trzysiek Feb 11, 2025
2e93592
Merge branch 'main' into datetime
trzysiek Feb 11, 2025
23e9da1
Add ctx
trzysiek Feb 12, 2025
e16790d
Cleanup
trzysiek Feb 12, 2025
0ba12ab
Merge branch 'main' into datetime
trzysiek Feb 24, 2025
5e6b31f
Fixed tests?
trzysiek Feb 24, 2025
e9fe338
Quickfix interface signature
trzysiek Feb 24, 2025
d96872d
Merge branch 'main' into datetime
trzysiek Mar 3, 2025
8a4afb7
Merge branch 'main' into datetime
Mar 5, 2025
421d1b7
fix some compilation errors
Mar 5, 2025
a535687
wip
Mar 5, 2025
e82c020
Merge branch 'main' into datetime
Mar 9, 2025
ab8ffae
Some fixes. Needs 1 improvement...
Mar 9, 2025
7e52e9f
Merge branch 'main' into datetime
Mar 10, 2025
a15621a
Better design - to debug
Mar 10, 2025
0e630e9
WORKS (tests)
Mar 10, 2025
40c84a6
All tests pass. Lol!
Mar 10, 2025
af6d1bb
Fix smoke test
Mar 10, 2025
04741cf
Quickfix fixing 2 tests
Mar 10, 2025
6781b0c
Add debug
Mar 10, 2025
33f1f9c
more debug
Mar 10, 2025
6121144
More debug
Mar 10, 2025
26d6b04
hacky-fix bug?
Mar 10, 2025
8c68d3b
Merge branch 'main' into datetime
Mar 11, 2025
af1cdc6
Some cleanup start
Mar 11, 2025
ca689e8
Merge branch 'main' into datetime
Mar 13, 2025
aee3be2
fix frontend_connectors tests
Mar 13, 2025
f9d79e2
fix all tests
Mar 13, 2025
357325d
fixed?
Mar 13, 2025
49fb4c8
fix all tests
Mar 13, 2025
88a9941
Cleanup
Mar 14, 2025
baa8be3
Merge branch 'main' into datetime
Mar 16, 2025
67befce
Cleanup
Mar 16, 2025
c918b82
Cleanup 2
Mar 16, 2025
43fbcb2
Cleanup 3
Mar 16, 2025
389baa1
Let's fix one last new test
trzysiek Mar 18, 2025
a904ac7
wip
trzysiek Mar 19, 2025
6e4ab64
Merge branch 'main' into datetime
trzysiek Mar 21, 2025
91258de
Merge branch 'main' into datetime
trzysiek Mar 22, 2025
a60d630
cleanup
trzysiek Mar 22, 2025
7c3d5f7
helper functions
trzysiek Mar 22, 2025
ecfd011
Merge branch 'main' into datetime
trzysiek Mar 23, 2025
9b9336d
Merge branch 'main' into datetime
trzysiek Mar 24, 2025
aac2857
WIP i think
trzysiek Mar 25, 2025
48cc0a7
Merge branch 'main' into datetime
trzysiek Mar 26, 2025
7456f14
fix test
trzysiek Mar 26, 2025
a4cc655
Merge branch 'main' into datetime
trzysiek Mar 29, 2025
0327c0a
Merge branch 'main' into datetime
pdelewski May 9, 2025
7c8c91a
Removing unused applyFieldMapSyntax
pdelewski May 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/v2_test_objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -422,8 +422,8 @@ func (p *QueryTransformationPipeline) ComposeResult(results [][]model.QueryResul
type QueryTransformer1 struct {
}

func (p *QueryTransformer1) Transform(queries []*model.Query) ([]*model.Query, error) {
logger.Debug().Msg("SimpleQueryTransformationPipeline: Transform")
func (p *QueryTransformer1) Transform(ctx context.Context, queries []*model.Query) ([]*model.Query, error) {
logger.DebugWithCtx(ctx).Msg("SimpleQueryTransformationPipeline: Transform")
// Do basic transformation

return queries, nil
Expand Down
19 changes: 15 additions & 4 deletions platform/clickhouse/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,6 @@ const (
Invalid
)

func (c *Column) String() string {
return fmt.Sprintf("%s %s", c.Name, c.Type.String())
}

func (t BaseType) String() string {
return t.Name
}
Expand Down Expand Up @@ -326,6 +322,21 @@ func NewEmptyTable(tableName string) *Table {
return &Table{Name: tableName, Config: NewChTableConfigNoAttrs()}
}

func (col *Column) String() string {
return fmt.Sprintf("%s %s", col.Name, col.Type.String())
}

// IsDatetime <=> is it DateTime or Date (but NOT DateTime64)
func (col *Column) IsDatetime() bool {
isDatetime := strings.HasPrefix(col.Type.String(), "DateTime") || strings.HasPrefix(col.Type.String(), "Date")
isDatetime64 := strings.HasPrefix(col.Type.String(), "DateTime64")
return isDatetime && !isDatetime64
}

func (col *Column) IsDatetime64() bool {
return strings.HasPrefix(col.Type.String(), "DateTime64")
}

func (col *Column) isArray() bool {
return col.Type.isArray()
}
Expand Down
88 changes: 0 additions & 88 deletions platform/clickhouse/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,7 @@
package clickhouse

import (
"bytes"
"fmt"
"github.com/QuesmaOrg/quesma/platform/logger"
"github.com/QuesmaOrg/quesma/platform/model"
"github.com/goccy/go-json"
"strings"
"time"
)

// Code doesn't need to be pretty, 99.9% it's just for our purposes
Expand Down Expand Up @@ -103,85 +97,3 @@ func parseTypeFromShowColumns(typ, name string) (Type, string) {
}
return parseTypeRec(typ, name)
}

func PrettyJson(jsonStr string) string {
var prettyJSON bytes.Buffer
if err := json.Indent(&prettyJSON, []byte(jsonStr), "", " "); err != nil {
return fmt.Sprintf("PrettyJson err: %v\n", err)
}
return prettyJSON.String()
}

// TimestampGroupBy returns string to be used in the select part of Clickhouse query, when grouping by timestamp interval.
// e.g.
// - timestampGroupBy("@timestamp", DateTime64, 30 seconds) --> toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)
// - timestampGroupBy("@timestamp", DateTime, 30 seconds) --> toInt64(toUnixTimestamp(`@timestamp`)/30)
func TimestampGroupBy(timestampField model.Expr, typ DateTimeType, groupByInterval time.Duration) model.Expr {

createAExp := func(innerFuncName string, interval int64) model.Expr {
toUnixTsFunc := model.NewInfixExpr(
model.NewFunction(innerFuncName, timestampField),
" / ", // TODO nasty hack to make our string-based tests pass. Operator should not contain spaces obviously
model.NewLiteral(interval))
return model.NewFunction("toInt64", toUnixTsFunc)
}

switch typ {
case DateTime64:
// as string: fmt.Sprintf("toInt64(toUnixTimestamp(`%s`)/%f)", timestampFieldName, groupByInterval.Seconds())
return createAExp("toUnixTimestamp64Milli", groupByInterval.Milliseconds())
case DateTime:
return createAExp("toUnixTimestamp", groupByInterval.Milliseconds()/1000)
default:
logger.Error().Msgf("invalid timestamp fieldname: %s", timestampFieldName)
return model.NewLiteral("invalid") // maybe create new type InvalidExpr?
}
}

func TimestampGroupByWithTimezone(timestampField model.Expr, typ DateTimeType,
groupByInterval time.Duration, timezone string) model.Expr {

// If no timezone, or timezone is default (UTC), we just return TimestampGroupBy(...)
if timezone == "" {
return TimestampGroupBy(timestampField, typ, groupByInterval)
}

createAExp := func(innerFuncName string, interval, offsetMultiplier int64) model.Expr {
var offset model.Expr
offset = model.NewFunction(
"timeZoneOffset",
model.NewFunction(
"toTimezone",
timestampField, model.NewLiteral("'"+timezone+"'"),
),
)
if offsetMultiplier != 1 {
offset = model.NewInfixExpr(offset, "*", model.NewLiteral(offsetMultiplier))
}

unixTsWithOffset := model.NewInfixExpr(
model.NewFunction(innerFuncName, timestampField),
"+",
offset,
)

groupByExpr := model.NewInfixExpr(
model.NewParenExpr(unixTsWithOffset),
" / ", // TODO nasty hack to make our string-based tests pass. Operator should not contain spaces obviously
model.NewLiteral(interval),
)

return model.NewFunction("toInt64", groupByExpr)
}

switch typ {
case DateTime64:
// e.g: (toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone("timestamp",'Europe/Warsaw'))*1000) / 600000
return createAExp("toUnixTimestamp64Milli", groupByInterval.Milliseconds(), 1000)
case DateTime:
return createAExp("toUnixTimestamp", groupByInterval.Milliseconds()/1000, 1)
default:
logger.Error().Msgf("invalid timestamp fieldname: %s", timestampFieldName)
return model.NewLiteral("invalid") // maybe create new type InvalidExpr?
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package frontend_connectors

import (
"context"
"fmt"
"github.com/QuesmaOrg/quesma/platform/logger"
"github.com/QuesmaOrg/quesma/platform/model"
Expand Down Expand Up @@ -160,7 +161,7 @@ func (s searchAfterStrategyBasicAndFast) transform(query *model.Query, searchAft
return query, nil
}

func (s *SchemaCheckPass) applySearchAfterParameter(indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
func (s *SchemaCheckPass) applySearchAfterParameter(ctx context.Context, indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
searchAfterParsed, err := s.searchAfterStrategy.validateAndParse(query, indexSchema)
if err != nil {
return nil, err
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package frontend_connectors

import (
"context"
"fmt"
"github.com/QuesmaOrg/quesma/platform/clickhouse"
"github.com/QuesmaOrg/quesma/platform/config"
Expand Down Expand Up @@ -157,7 +158,7 @@ func Test_applySearchAfterParameter(t *testing.T) {
tc.transformedQueryExpected.SearchAfter = tc.searchAfter

transformer := NewSchemaCheckPass(&config.QuesmaConfiguration{IndexConfig: indexConfig}, tableDiscovery, strategy)
actual, err := transformer.applySearchAfterParameter(Schema, tc.query)
actual, err := transformer.applySearchAfterParameter(context.Background(), Schema, tc.query)
assert.Equal(t, tc.errorExpected, err != nil, "Expected error: %v, got: %v", tc.errorExpected, err)
if err == nil {
assert.Equal(t,
Expand Down
Loading
Loading