Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions platform/clickhouse/clickhouse_schema.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright Quesma, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0
package clickhouse

import (
"reflect"
"time"
)

// this is catch all type for all types we do not exlicitly support
type UnknownType struct{}

type ClickhouseTypeResolver struct{}

func (r *ClickhouseTypeResolver) ResolveType(clickHouseTypeName string) reflect.Type {
switch clickHouseTypeName {
case "String", "LowCardinality(String)", "UUID", "FixedString":
return reflect.TypeOf("")
case "DateTime64", "DateTime", "Date", "DateTime64(3)":
return reflect.TypeOf(time.Time{})
case "UInt8", "UInt16", "UInt32", "UInt64":
return reflect.TypeOf(uint64(0))
case "Int8", "Int16", "Int32":
return reflect.TypeOf(int32(0))
case "Int64":
return reflect.TypeOf(int64(0))
case "Float32", "Float64":
return reflect.TypeOf(float64(0))
case "Point":
return reflect.TypeOf(Point{})
case "Bool":
return reflect.TypeOf(true)
case "JSON":
return reflect.TypeOf(map[string]interface{}{})
case "Map(String, Nullable(String))", "Map(String, String)", "Map(LowCardinality(String), String)", "Map(LowCardinality(String), Nullable(String))":
return reflect.TypeOf(map[string]string{})
case "Unknown":
return reflect.TypeOf(UnknownType{})
}
return nil
}
53 changes: 24 additions & 29 deletions platform/database_common/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"github.com/QuesmaOrg/quesma/platform/clickhouse"
"github.com/QuesmaOrg/quesma/platform/config"
"github.com/QuesmaOrg/quesma/platform/doris"
"github.com/QuesmaOrg/quesma/platform/logger"
"github.com/QuesmaOrg/quesma/platform/schema"
"github.com/QuesmaOrg/quesma/platform/util"
Expand Down Expand Up @@ -211,44 +212,38 @@ func (t MultiValueType) GetColumn(name string) *Column {
}

func NewBaseType(clickHouseTypeName string) BaseType {
var GoType = ResolveType(clickHouseTypeName)
// TODO: currently, NewBaseType is only used in tests or create table or insert, not in Doris's code, so the ClickHouse schema is used here.
var r TypeResolver = &clickhouse.ClickhouseTypeResolver{}
var GoType = r.ResolveType(clickHouseTypeName)
if GoType == nil {
// default, probably good for dates, etc.
GoType = reflect.TypeOf("")
}
return BaseType{Name: clickHouseTypeName, GoType: GoType}
}

// this is catch all type for all types we do not exlicitly support
type UnknownType struct{}

func ResolveType(clickHouseTypeName string) reflect.Type {
switch clickHouseTypeName {
case "String", "LowCardinality(String)", "UUID", "FixedString":
return reflect.TypeOf("")
case "DateTime64", "DateTime", "Date", "DateTime64(3)":
return reflect.TypeOf(time.Time{})
case "UInt8", "UInt16", "UInt32", "UInt64":
return reflect.TypeOf(uint64(0))
case "Int8", "Int16", "Int32":
return reflect.TypeOf(int32(0))
case "Int64":
return reflect.TypeOf(int64(0))
case "Float32", "Float64":
return reflect.TypeOf(float64(0))
case "Point":
return reflect.TypeOf(clickhouse.Point{})
case "Bool":
return reflect.TypeOf(true)
case "JSON":
return reflect.TypeOf(map[string]interface{}{})
case "Map(String, Nullable(String))", "Map(String, String)", "Map(LowCardinality(String), String)", "Map(LowCardinality(String), Nullable(String))":
return reflect.TypeOf(map[string]string{})
case "Unknown":
return reflect.TypeOf(UnknownType{})
func NewBaseTypeWithInstanceName(typeName string, instanceName string) BaseType {
Copy link
Contributor

@pdelewski pdelewski Jul 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be better to use some const enum instead of string as instanceName

r := GetTypeResolver(instanceName)
var GoType = r.ResolveType(typeName)
if GoType == nil {
// default, probably good for dates, etc.
GoType = reflect.TypeOf("")
}
return BaseType{Name: typeName, GoType: GoType}
}

func GetTypeResolver(instanceName string) TypeResolver {
var r TypeResolver
if instanceName == "doris" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As above, const enum would be better IMO

r = &doris.DorisTypeResolver{}
} else {
r = &clickhouse.ClickhouseTypeResolver{}
}
return r
}

return nil
type TypeResolver interface {
ResolveType(typeName string) reflect.Type
}

// 'value': value of a field, from unmarshalled JSON
Expand Down
4 changes: 4 additions & 0 deletions platform/database_common/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ func TestGetDateTimeType(t *testing.T) {
"timestamp2": {Name: "timestamp2", Type: NewBaseType("DateTime('UTC')")},
"timestamp64_1": {Name: "timestamp64_1", Type: NewBaseType("DateTime64")},
"timestamp64_2": {Name: "timestamp64_2", Type: NewBaseType("DateTime64(3, 'UTC')")},
"datetime1": {Name: "datetime1", Type: NewBaseType("datetime")},
"date1": {Name: "date1", Type: NewBaseType("date")},
},
Config: NewChTableConfigTimestampStringAttr(),
}
Expand All @@ -25,5 +27,7 @@ func TestGetDateTimeType(t *testing.T) {
assert.Equal(t, DateTime64, table.GetDateTimeType(ctx, "timestamp64_1", true))
assert.Equal(t, DateTime64, table.GetDateTimeType(ctx, "timestamp64_2", true))
assert.Equal(t, DateTime64, table.GetDateTimeType(ctx, timestampFieldName, true)) // default, created by us
assert.Equal(t, DateTime64, table.GetDateTimeType(ctx, "datetime1", true))
assert.Equal(t, DateTime, table.GetDateTimeType(ctx, "date1", true))
assert.Equal(t, Invalid, table.GetDateTimeType(ctx, "non-existent", false))
}
4 changes: 2 additions & 2 deletions platform/database_common/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ func (t *Table) GetDateTimeType(ctx context.Context, fieldName string, dateInSch
if col, ok := t.Cols[fieldName]; ok {
typeName := col.Type.String()
// hasPrefix, not equal, because we can have DateTime64(3) and we want to catch it
if strings.HasPrefix(typeName, "DateTime64") {
if strings.HasPrefix(typeName, "DateTime64") || strings.HasPrefix(typeName, "datetime") {
return DateTime64
}
if strings.HasPrefix(typeName, "DateTime") {
if strings.HasPrefix(typeName, "DateTime") || strings.HasPrefix(typeName, "date") {
return DateTime
}
}
Expand Down
20 changes: 11 additions & 9 deletions platform/database_common/table_discovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,7 @@ func (td *tableDiscovery) autoConfigureTables(tables map[string]map[string]colum

func (td *tableDiscovery) populateTableDefinitions(configuredTables map[string]discoveredTable, databaseName string, cfg *config.QuesmaConfiguration) {

instanceName := td.dbConnPool.InstanceName()
tableMap := NewTableMap()
for tableName, resTable := range configuredTables {
var columnsMap = make(map[string]*Column)
Expand All @@ -393,7 +394,7 @@ func (td *tableDiscovery) populateTableDefinitions(configuredTables map[string]d
}
}

column := ResolveColumn(col, columnMeta.colType)
column := ResolveColumn(col, columnMeta.colType, instanceName)
if column != nil {
column.Comment = columnMeta.comment
column.Origin = columnMeta.origin
Expand Down Expand Up @@ -481,12 +482,13 @@ func (td *tableDiscovery) TableDefinitions() *TableMap {
return td.tableDefinitions.Load()
}

func ResolveColumn(colName, colType string) *Column {
func ResolveColumn(colName, colType string, instanceName string) *Column {
isNullable := false
if isNullableType(colType) {
isNullable = true
colType = strings.TrimSuffix(strings.TrimPrefix(colType, "Nullable("), ")")
}
r := GetTypeResolver(instanceName)

if isArrayType(colType) {
arrayType := strings.TrimSuffix(strings.TrimPrefix(colType, "Array("), ")")
Expand All @@ -495,7 +497,7 @@ func ResolveColumn(colName, colType string) *Column {
arrayType = strings.TrimSuffix(strings.TrimPrefix(arrayType, "Nullable("), ")")
}
if isArrayType(arrayType) {
innerColumn := ResolveColumn("inner", arrayType)
innerColumn := ResolveColumn("inner", arrayType, instanceName)
if innerColumn == nil {
logger.Warn().Msgf("invalid inner array type for column %s, %s", colName, colType)
return nil
Expand All @@ -508,7 +510,7 @@ func ResolveColumn(colName, colType string) *Column {
},
}
}
GoType := ResolveType(arrayType)
GoType := r.ResolveType(arrayType)
if GoType != nil {
return &Column{
Name: colName,
Expand All @@ -518,7 +520,7 @@ func ResolveColumn(colName, colType string) *Column {
},
}
} else if isTupleType(arrayType) {
tupleColumn := ResolveColumn("Tuple", arrayType)
tupleColumn := ResolveColumn("Tuple", arrayType, instanceName)
if tupleColumn == nil {
logger.Warn().Msgf("invalid tuple type for column %s, %s", colName, colType)
return nil
Expand Down Expand Up @@ -558,7 +560,7 @@ func ResolveColumn(colName, colType string) *Column {
Name: colName,
Type: BaseType{
Name: "Int32",
GoType: NewBaseType("Int32").GoType,
GoType: NewBaseTypeWithInstanceName("Int32", instanceName).GoType,
},
}
}
Expand All @@ -567,12 +569,12 @@ func ResolveColumn(colName, colType string) *Column {
if strings.HasPrefix(colType, "DateTime") {
colType = removePrecision(colType)
}
if GoType := ResolveType(colType); GoType != nil {
if GoType := r.ResolveType(colType); GoType != nil {
return &Column{
Name: colName,
Type: BaseType{
Name: colType,
GoType: NewBaseType(colType).GoType,
GoType: NewBaseTypeWithInstanceName(colType, instanceName).GoType,
Nullable: isNullable,
},
}
Expand All @@ -583,7 +585,7 @@ func ResolveColumn(colName, colType string) *Column {
Name: colName,
Type: BaseType{
Name: typeName,
GoType: NewBaseType("Unknown").GoType,
GoType: NewBaseTypeWithInstanceName("Unknown", instanceName).GoType,
Nullable: isNullable,
},
}
Expand Down
4 changes: 2 additions & 2 deletions platform/database_common/table_discovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ func Test_resolveColumn(t *testing.T) {

for i, tt := range tests {
t.Run(util.PrettyTestName(tt.name, i), func(t *testing.T) {
assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType)
assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType, "clickhouse"), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType)
})
}
}
Expand Down Expand Up @@ -300,7 +300,7 @@ func Test_resolveColumn_Nullable(t *testing.T) {

for i, tt := range tests {
t.Run(util.PrettyTestName(tt.name, i), func(t *testing.T) {
assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType)
assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType, "clickhouse"), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType)
})
}
}
Expand Down
53 changes: 53 additions & 0 deletions platform/doris/doris_schema.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright Quesma, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0
package doris

import (
"reflect"
"strings"
"time"
)

// this is catch all type for all types we do not exlicitly support
type UnknownType struct{}

type DorisTypeResolver struct{}

func (r *DorisTypeResolver) ResolveType(dorisTypeName string) reflect.Type {
dorisTypeName = strings.ToLower(dorisTypeName)
switch dorisTypeName {
case "char", "varchar", "string", "text":
return reflect.TypeOf("")
case "date", "datetime", "datev2", "datetimev2":
return reflect.TypeOf(time.Time{})
case "tinyint", "smallint", "int":
return reflect.TypeOf(int32(0))
case "bigint":
return reflect.TypeOf(int64(0))
case "largeint":
return reflect.TypeOf("") // LargeInt is typically handled as string due to size
case "boolean":
return reflect.TypeOf(true)
case "float":
return reflect.TypeOf(float32(0))
case "double":
return reflect.TypeOf(float64(0))
case "decimal", "decimalv2":
return reflect.TypeOf("") // Decimals often handled as strings for precision
case "json":
return reflect.TypeOf(map[string]interface{}{})
case "hll":
return reflect.TypeOf([]byte{}) // HLL is a binary type
case "bitmap":
return reflect.TypeOf([]byte{}) // Bitmap is also binary
case "array":
return reflect.TypeOf([]interface{}{})
case "map":
return reflect.TypeOf(map[string]interface{}{})
case "struct":
return reflect.TypeOf(map[string]interface{}{})
case "Unknown":
return reflect.TypeOf(UnknownType{})
}
return nil
}
2 changes: 1 addition & 1 deletion platform/ingest/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ func (ip *IngestProcessor) createTableObject(tableName string, columnsFromJson [
// Remove DEFAULT clause from the type
colType = strings.Split(colType, " DEFAULT")[0]
}
resCol := database_common.ResolveColumn(name, colType)
resCol := database_common.ResolveColumn(name, colType, "clickhouse")
return resCol.Type
}

Expand Down
Loading