Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 50792b1

Browse files
authored
Resolve table name with schema (#768)
1 parent 4352fca commit 50792b1

File tree

12 files changed

+66
-35
lines changed

12 files changed

+66
-35
lines changed

quesma/clickhouse/clickhouse.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ func (lm *LogManager) Stop() {
115115

116116
type discoveredTable struct {
117117
name string
118+
databaseName string
118119
columnTypes map[string]string
119120
config config.IndexConfiguration
120121
comment string

quesma/clickhouse/quesma_communicator.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,10 @@ func executeQuery(ctx context.Context, lm *LogManager, query *model.Query, field
160160
//
161161

162162
settings := make(clickhouse.Settings)
163-
settings["readonly"] = "1"
163+
// this "readonly" setting turned out to be causing problems with Hydrolix queries
164+
// the queries looked pretty legit, but the key difference was the use of schema (`FROM "schema"."tableName"`)
165+
// to be revisited in the future
166+
// settings["readonly"] = "1"
164167
settings["allow_ddl"] = "0"
165168

166169
if query.OptimizeHints != nil {
@@ -185,7 +188,6 @@ func executeQuery(ctx context.Context, lm *LogManager, query *model.Query, field
185188
performanceResult.Error = err
186189
return nil, performanceResult, end_user_errors.GuessClickhouseErrorType(err).InternalDetails("clickhouse: query failed. err: %v, query: %v", err, queryAsString)
187190
}
188-
189191
res, err = read(rows, fields, rowToScan)
190192

191193
elapsed := span.End(nil)

quesma/clickhouse/table_discovery.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ func (t TableDiscoveryTableProviderAdapter) TableDefinitions() map[string]schema
7878
Type: column.Type.String(),
7979
}
8080
}
81+
table.DatabaseName = value.DatabaseName
8182
tables[tableName] = table
8283
return true
8384
})
@@ -162,7 +163,7 @@ func (td *tableDiscovery) configureTables(tables map[string]map[string]string, d
162163
comment := td.tableComment(databaseName, table)
163164
createTableQuery := td.createTableQuery(databaseName, table)
164165
// we assume here that @timestamp field is always present in the table, or it's explicitly configured
165-
configuredTables[table] = discoveredTable{table, columns, indexConfig, comment, createTableQuery, ""}
166+
configuredTables[table] = discoveredTable{table, databaseName, columns, indexConfig, comment, createTableQuery, ""}
166167
}
167168
} else {
168169
notConfiguredTables = append(notConfiguredTables, table)
@@ -191,7 +192,7 @@ func (td *tableDiscovery) autoConfigureTables(tables map[string]map[string]strin
191192
} else {
192193
maybeTimestampField = td.tableTimestampField(databaseName, table, ClickHouse)
193194
}
194-
configuredTables[table] = discoveredTable{table, columns, config.IndexConfiguration{}, comment, createTableQuery, maybeTimestampField}
195+
configuredTables[table] = discoveredTable{table, databaseName, columns, config.IndexConfiguration{}, comment, createTableQuery, maybeTimestampField}
195196

196197
}
197198
for tableName, table := range configuredTables {

quesma/model/expr.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,10 @@ func (s DistinctExpr) Accept(v ExprVisitor) interface{} {
127127
type TableRef struct {
128128
Name string
129129
// to be considered - alias (e.g. FROM tableName AS t)
130-
// to be considered - database prefix (e.g. FROM databaseName.tableName)
130+
131+
// DatabaseName is optional and represents what in database realm is called 'schema', e.g. 'FROM databaseName.tableName'
132+
// ClickHouse calls this 'database' so we stick to that; FWIW - Hydrolix calls this a 'project'.
133+
DatabaseName string
131134
}
132135

133136
func NewTableRef(name string) TableRef {
@@ -138,6 +141,10 @@ func (t TableRef) Accept(v ExprVisitor) interface{} {
138141
return v.VisitTableRef(t)
139142
}
140143

144+
func NewTableRefWithDatabaseName(name, databaseName string) TableRef {
145+
return TableRef{Name: name, DatabaseName: databaseName}
146+
}
147+
141148
type OrderByDirection int8
142149

143150
const (

quesma/model/expr_string_renderer.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,22 @@ func (v *renderer) VisitDistinctExpr(e DistinctExpr) interface{} {
107107
}
108108

109109
func (v *renderer) VisitTableRef(e TableRef) interface{} {
110+
var result []string
110111

112+
if e.DatabaseName != "" {
113+
if identifierRegexp.MatchString(e.DatabaseName) {
114+
result = append(result, e.DatabaseName)
115+
} else {
116+
result = append(result, strconv.Quote(e.DatabaseName))
117+
}
118+
}
111119
if identifierRegexp.MatchString(e.Name) {
112-
return e.Name
120+
result = append(result, e.Name)
121+
} else {
122+
result = append(result, strconv.Quote(e.Name))
113123
}
114124

115-
return strconv.Quote(e.Name)
125+
return strings.Join(result, ".")
116126
}
117127

118128
func (v *renderer) VisitAliasedExpr(e AliasedExpr) interface{} {

quesma/quesma/mappings_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func newSchemaFromColumns(fields map[string]schema.Column) schema.Schema {
104104
Type: parsedType,
105105
}
106106
}
107-
return schema.NewSchema(schemaFields, true)
107+
return schema.NewSchema(schemaFields, true, "")
108108
}
109109

110110
func TestParseMappings_KibanaSampleFlights(t *testing.T) {

quesma/quesma/schema_transformer.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ func (s *SchemaCheckPass) applyPhysicalFromExpression(currentSchema schema.Schem
347347
}
348348

349349
// TODO compute physical from expression based on single table or union or whatever ....
350-
physicalFromExpression := model.NewTableRef(query.TableName)
350+
physicalFromExpression := model.NewTableRefWithDatabaseName(query.TableName, currentSchema.DatabaseName)
351351

352352
visitor := model.NewBaseVisitor()
353353

@@ -362,6 +362,7 @@ func (s *SchemaCheckPass) applyPhysicalFromExpression(currentSchema schema.Schem
362362
if _, ok := expr.(*model.SelectCommand); ok {
363363
query.SelectCommand = *expr.(*model.SelectCommand)
364364
}
365+
365366
return query, nil
366367
}
367368

quesma/schema/hierarchical_schema_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ func Test_SchemaToHierarchicalSchema(t *testing.T) {
1515
"product.product_id": {PropertyName: "product.product_id", InternalPropertyName: "product::product_id", Type: QuesmaTypeInteger},
1616
"triple.nested.example1": {PropertyName: "triple.nested.example1", InternalPropertyName: "triple::nested::example1", Type: QuesmaTypeText},
1717
"triple.nested.example2": {PropertyName: "triple.nested.example2", InternalPropertyName: "triple::nested::example2", Type: QuesmaTypeKeyword},
18-
}, map[FieldName]FieldName{}, true)
18+
}, map[FieldName]FieldName{}, true, "")
1919

2020
hs := SchemaToHierarchicalSchema(&s)
2121
assert.Equal(t, "", hs.Name)

quesma/schema/registry.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ type (
2929
AutodiscoveryEnabled() bool
3030
}
3131
Table struct {
32-
Columns map[string]Column
32+
Columns map[string]Column
33+
DatabaseName string
3334
}
3435
Column struct {
3536
Name string
@@ -41,10 +42,10 @@ func (s *schemaRegistry) loadSchemas() (map[TableName]Schema, error) {
4142
definitions := s.dataSourceTableProvider.TableDefinitions()
4243
schemas := make(map[TableName]Schema)
4344
if s.dataSourceTableProvider.AutodiscoveryEnabled() {
44-
for tableName := range definitions {
45+
for tableName, table := range definitions {
4546
fields := make(map[FieldName]Field)
4647
existsInDataSource := s.populateSchemaFromTableDefinition(definitions, tableName, fields)
47-
schemas[TableName(tableName)] = NewSchema(fields, existsInDataSource)
48+
schemas[TableName(tableName)] = NewSchema(fields, existsInDataSource, table.DatabaseName)
4849
}
4950
return schemas, nil
5051
}
@@ -59,7 +60,11 @@ func (s *schemaRegistry) loadSchemas() (map[TableName]Schema, error) {
5960
s.populateAliases(indexConfiguration, fields, aliases)
6061
s.removeIgnoredFields(indexConfiguration, fields, aliases)
6162
s.removeGeoPhysicalFields(fields)
62-
schemas[TableName(indexName)] = NewSchemaWithAliases(fields, aliases, existsInDataSource)
63+
if tableDefinition, ok := definitions[indexName]; ok {
64+
schemas[TableName(indexName)] = NewSchemaWithAliases(fields, aliases, existsInDataSource, tableDefinition.DatabaseName)
65+
} else {
66+
schemas[TableName(indexName)] = NewSchemaWithAliases(fields, aliases, existsInDataSource, "")
67+
}
6368
}
6469

6570
return schemas, nil

quesma/schema/registry_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
4848
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword, InternalPropertyType: "String"},
4949
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
5050
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
51-
true),
51+
true, ""),
5252
found: true,
5353
},
5454
{
@@ -77,7 +77,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
7777
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
7878
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
7979

80-
true),
80+
true, ""),
8181
found: true,
8282
},
8383
{
@@ -104,7 +104,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
104104
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword, InternalPropertyType: ""},
105105
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
106106
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
107-
true),
107+
true, ""),
108108
found: true,
109109
},
110110
{
@@ -129,7 +129,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
129129
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword},
130130
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
131131
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
132-
true),
132+
true, ""),
133133
found: true,
134134
},
135135
{
@@ -145,7 +145,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
145145
},
146146
tableDiscovery: fixedTableProvider{tables: map[string]schema.Table{}},
147147
tableName: "some_table",
148-
want: schema.NewSchema(map[schema.FieldName]schema.Field{"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword}}, false),
148+
want: schema.NewSchema(map[schema.FieldName]schema.Field{"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword}}, false, ""),
149149
found: true,
150150
},
151151
{
@@ -171,7 +171,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
171171
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword},
172172
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
173173
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
174-
true),
174+
true, ""),
175175
found: true,
176176
},
177177
{
@@ -199,7 +199,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
199199
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
200200
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}}, map[schema.FieldName]schema.FieldName{
201201
"message_alias": "message",
202-
}, true),
202+
}, true, ""),
203203
found: true,
204204
},
205205
{
@@ -229,7 +229,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
229229
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
230230
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}}, map[schema.FieldName]schema.FieldName{
231231
"message_alias": "message",
232-
}, true),
232+
}, true, ""),
233233
found: true,
234234
},
235235
{
@@ -299,7 +299,7 @@ func Test_schemaRegistry_UpdateDynamicConfiguration(t *testing.T) {
299299
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.QuesmaTypeKeyword, InternalPropertyType: "String"},
300300
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
301301
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"}},
302-
true)
302+
true, "")
303303
resultSchema, resultFound := s.FindSchema(schema.TableName(tableName))
304304
assert.True(t, resultFound, "schema not found")
305305
if !reflect.DeepEqual(resultSchema, expectedSchema) {
@@ -320,7 +320,7 @@ func Test_schemaRegistry_UpdateDynamicConfiguration(t *testing.T) {
320320
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.QuesmaTypeTimestamp, InternalPropertyType: "DateTime64"},
321321
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.QuesmaTypeLong, InternalPropertyType: "Int64"},
322322
"new_column": {PropertyName: "new_column", InternalPropertyName: "new_column", Type: schema.QuesmaTypeText}},
323-
true)
323+
true, "")
324324
resultSchema, resultFound = s.FindSchema(schema.TableName(tableName))
325325
assert.True(t, resultFound, "schema not found")
326326
if !reflect.DeepEqual(resultSchema, expectedSchema) {

0 commit comments

Comments
 (0)