Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 1e0eb3e

Browse files
mieciumieciu
authored
Fix resolving indices when tables are auto-discovered (#765)
Users who haven't specified any table configurations aren't able to create data views today. 1. Because `_resolve` endpoint relies on schema registry, which - as turned out - has never been populated when the index configuration has been missing. 2. Because `_field_caps` were **not** routed properly. `matchedAgainstPattern` did rely only on config, not on schema registry, so it couldn't route queries to auto discovered tables. Therefore, timestamp field list has not been expanding and data view could not have been created. This PR addresses both issues. **How it works now:** <img width="1173" alt="image" src="https://github.com/user-attachments/assets/5845efb1-ecc1-4d56-86d0-1438c9cb888e">
1 parent 54d168a commit 1e0eb3e

File tree

11 files changed

+60
-31
lines changed

11 files changed

+60
-31
lines changed

‎quesma/clickhouse/table_discovery.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ type TableDiscovery interface {
3636
LastAccessTime() time.Time
3737
LastReloadTime() time.Time
3838
ForceReloadCh() <-chan chan<- struct{}
39+
AutodiscoveryEnabled() bool
3940
}
4041

4142
type tableDiscovery struct {
@@ -100,8 +101,8 @@ func (td *tableDiscovery) TableDefinitionsFetchError() error {
100101
return td.ReloadTablesError
101102
}
102103

103-
func (td *tableDiscovery) TableAutodiscoveryEnabled() bool {
104-
return len(td.cfg.IndexConfig) == 0
104+
func (td *tableDiscovery) AutodiscoveryEnabled() bool {
105+
return td.cfg.IndexAutodiscoveryEnabled()
105106
}
106107

107108
func (td *tableDiscovery) LastAccessTime() time.Time {
@@ -138,7 +139,7 @@ func (td *tableDiscovery) ReloadTableDefinitions() {
138139
td.tableDefinitionsLastReloadUnixSec.Store(time.Now().Unix())
139140
return
140141
} else {
141-
if td.TableAutodiscoveryEnabled() {
142+
if td.AutodiscoveryEnabled() {
142143
configuredTables = td.autoConfigureTables(tables, databaseName)
143144
} else {
144145
configuredTables = td.configureTables(tables, databaseName)

‎quesma/quesma/config/config.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -351,11 +351,6 @@ func (c *QuesmaConfiguration) validateSchemaConfiguration(config IndexConfigurat
351351
return err
352352
}
353353

354-
//func countPrimaryKeys(config IndexConfiguration) (count int) {
355-
// for _, configuration := range config.SchemaOverrides.Fields {
356-
// if configuration.IsPrimaryKey {
357-
// count++
358-
// }
359-
// }
360-
// return count
361-
//}
354+
func (c *QuesmaConfiguration) IndexAutodiscoveryEnabled() bool {
355+
return len(c.IndexConfig) == 0
356+
}

‎quesma/quesma/matchers.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"quesma/quesma/config"
99
"quesma/quesma/mux"
1010
"quesma/quesma/types"
11+
"quesma/schema"
1112
"quesma/tracing"
1213
"strings"
1314
)
@@ -44,7 +45,7 @@ func matchedAgainstBulkBody(configuration *config.QuesmaConfiguration) mux.Reque
4445
})
4546
}
4647

47-
func matchedAgainstPattern(configuration *config.QuesmaConfiguration) mux.RequestMatcher {
48+
func matchedAgainstPattern(configuration *config.QuesmaConfiguration, sr schema.Registry) mux.RequestMatcher {
4849
return mux.RequestMatcherFunc(func(req *mux.Request) bool {
4950
indexPattern := elasticsearch.NormalizePattern(req.Params["index"])
5051
if elasticsearch.IsInternalIndex(indexPattern) {
@@ -61,7 +62,13 @@ func matchedAgainstPattern(configuration *config.QuesmaConfiguration) mux.Reques
6162
return false
6263
}
6364
}
64-
65+
if configuration.IndexAutodiscoveryEnabled() {
66+
for tableName := range sr.AllSchemas() {
67+
if config.MatchName(elasticsearch.NormalizePattern(indexPattern), string(tableName)) {
68+
return true
69+
}
70+
}
71+
}
6572
for _, pattern := range indexPatterns {
6673
for _, indexName := range configuration.IndexConfig {
6774
if config.MatchName(elasticsearch.NormalizePattern(pattern), indexName.Name) {
@@ -73,6 +80,13 @@ func matchedAgainstPattern(configuration *config.QuesmaConfiguration) mux.Reques
7380
}
7481
return false
7582
} else {
83+
if configuration.IndexAutodiscoveryEnabled() {
84+
for tableName := range sr.AllSchemas() {
85+
if config.MatchName(elasticsearch.NormalizePattern(indexPattern), string(tableName)) {
86+
return true
87+
}
88+
}
89+
}
7690
for _, index := range configuration.IndexConfig {
7791
pattern := elasticsearch.NormalizePattern(indexPattern)
7892
if config.MatchName(pattern, index.Name) {

‎quesma/quesma/router.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
9898
return resolveIndexResult(sources)
9999
})
100100

101-
router.Register(routes.IndexCountPath, and(method("GET"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
101+
router.Register(routes.IndexCountPath, and(method("GET"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
102102
cnt, err := queryRunner.handleCount(ctx, req.Params["index"])
103103
if err != nil {
104104
if errors.Is(quesma_errors.ErrIndexNotExists(), err) {
@@ -137,7 +137,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
137137
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
138138
})
139139

140-
router.Register(routes.IndexSearchPath, and(method("GET", "POST"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
140+
router.Register(routes.IndexSearchPath, and(method("GET", "POST"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
141141

142142
body, err := types.ExpectJSON(req.ParsedBody)
143143
if err != nil {
@@ -159,7 +159,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
159159
}
160160
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
161161
})
162-
router.Register(routes.IndexAsyncSearchPath, and(method("POST"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
162+
router.Register(routes.IndexAsyncSearchPath, and(method("POST"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
163163
waitForResultsMs := 1000 // Defaults to 1 second as in docs
164164
if v, ok := req.Params["wait_for_completion_timeout"]; ok {
165165
if w, err := time.ParseDuration(v); err == nil {
@@ -196,7 +196,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
196196
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
197197
})
198198

199-
router.Register(routes.IndexMappingPath, and(method("PUT"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
199+
router.Register(routes.IndexMappingPath, and(method("PUT"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
200200
index := req.Params["index"]
201201

202202
body, err := types.ExpectJSON(req.ParsedBody)
@@ -211,7 +211,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
211211
return putIndexResult(index)
212212
})
213213

214-
router.Register(routes.IndexMappingPath, and(method("GET"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
214+
router.Register(routes.IndexMappingPath, and(method("GET"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
215215
index := req.Params["index"]
216216

217217
foundSchema, found := sr.FindSchema(schema.TableName(index))
@@ -242,7 +242,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
242242
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
243243
})
244244

245-
router.Register(routes.FieldCapsPath, and(method("GET", "POST"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
245+
router.Register(routes.FieldCapsPath, and(method("GET", "POST"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
246246

247247
responseBody, err := field_capabilities.HandleFieldCaps(ctx, cfg, sr, req.Params["index"], lm)
248248
if err != nil {
@@ -257,7 +257,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
257257
}
258258
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
259259
})
260-
router.Register(routes.TermsEnumPath, and(method("POST"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
260+
router.Register(routes.TermsEnumPath, and(method("POST"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
261261
if strings.Contains(req.Params["index"], ",") {
262262
return nil, errors.New("multi index terms enum is not yet supported")
263263
} else {
@@ -278,7 +278,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
278278
}
279279
})
280280

281-
router.Register(routes.EQLSearch, and(method("GET", "POST"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
281+
router.Register(routes.EQLSearch, and(method("GET", "POST"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
282282
body, err := types.ExpectJSON(req.ParsedBody)
283283
if err != nil {
284284
return nil, err
@@ -295,7 +295,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
295295
return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil
296296
})
297297

298-
router.Register(routes.IndexPath, and(method("PUT"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
298+
router.Register(routes.IndexPath, and(method("PUT"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
299299
index := req.Params["index"]
300300

301301
body, err := types.ExpectJSON(req.ParsedBody)
@@ -315,7 +315,7 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
315315
return putIndexResult(index)
316316
})
317317

318-
router.Register(routes.IndexPath, and(method("GET"), matchedAgainstPattern(cfg)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
318+
router.Register(routes.IndexPath, and(method("GET"), matchedAgainstPattern(cfg, sr)), func(ctx context.Context, req *mux.Request) (*mux.Result, error) {
319319
index := req.Params["index"]
320320

321321
foundSchema, found := sr.FindSchema(schema.TableName(index))

‎quesma/quesma/router_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/stretchr/testify/assert"
77
"quesma/quesma/config"
88
"quesma/quesma/mux"
9+
"quesma/schema"
910
"testing"
1011
)
1112

@@ -131,8 +132,7 @@ func Test_matchedAgainstPattern(t *testing.T) {
131132
t.Run(tt.name, func(t *testing.T) {
132133

133134
req := &mux.Request{Params: map[string]string{"index": tt.pattern}, Body: tt.body}
134-
135-
assert.Equalf(t, tt.want, matchedAgainstPattern(&tt.configuration).Matches(req), "matchedAgainstPattern(%v)", tt.configuration)
135+
assert.Equalf(t, tt.want, matchedAgainstPattern(&tt.configuration, schema.StaticRegistry{}).Matches(req), "matchedAgainstPattern(%v)", tt.configuration)
136136
})
137137
}
138138
}

‎quesma/quesma/schema_transformer_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ func (f fixedTableProvider) TableDefinitions() map[string]schema.Table {
2020
return f.tables
2121
}
2222

23+
func (f fixedTableProvider) AutodiscoveryEnabled() bool { return false }
24+
2325
func Test_ipRangeTransform(t *testing.T) {
2426
const isIPAddressInRangePrimitive = "isIPAddressInRange"
2527
const CASTPrimitive = "CAST"

‎quesma/quesma/search.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ func (q *QueryRunner) executePlan(ctx context.Context, plan *model.ExecutionPlan
270270
}
271271

272272
func (q *QueryRunner) handleSearchCommon(ctx context.Context, indexPattern string, body types.JSON, optAsync *AsyncQuery, queryLanguage QueryLanguage) ([]byte, error) {
273-
sources, sourcesElastic, sourcesClickhouse := ResolveSources(indexPattern, q.cfg, q.im)
273+
sources, sourcesElastic, sourcesClickhouse := ResolveSources(indexPattern, q.cfg, q.im, q.schemaRegistry)
274274

275275
switch sources {
276276
case sourceBoth:

‎quesma/quesma/source_resolver.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"quesma/elasticsearch"
77
"quesma/logger"
88
"quesma/quesma/config"
9+
"quesma/schema"
910
"quesma/util"
1011
"slices"
1112
"strings"
@@ -18,7 +19,7 @@ const (
1819
sourceNone = "none"
1920
)
2021

21-
func ResolveSources(indexPattern string, cfg *config.QuesmaConfiguration, im elasticsearch.IndexManagement) (string, []string, []string) {
22+
func ResolveSources(indexPattern string, cfg *config.QuesmaConfiguration, im elasticsearch.IndexManagement, sr schema.Registry) (string, []string, []string) {
2223
if elasticsearch.IsIndexPattern(indexPattern) {
2324
matchesElastic := []string{}
2425
matchesClickhouse := []string{}
@@ -29,6 +30,13 @@ func ResolveSources(indexPattern string, cfg *config.QuesmaConfiguration, im ela
2930
matchesElastic = append(matchesElastic, indexName)
3031
}
3132
}
33+
if cfg.IndexAutodiscoveryEnabled() {
34+
for tableName := range sr.AllSchemas() {
35+
if config.MatchName(elasticsearch.NormalizePattern(indexPattern), string(tableName)) {
36+
matchesClickhouse = append(matchesClickhouse, string(tableName))
37+
}
38+
}
39+
}
3240

3341
for indexName, indexConfig := range cfg.IndexConfig {
3442
if util.IndexPatternMatches(pattern, indexName) && !indexConfig.Disabled {

‎quesma/quesma/source_resolver_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/stretchr/testify/assert"
77
"quesma/elasticsearch"
88
"quesma/quesma/config"
9+
"quesma/schema"
910
"quesma/util"
1011
"testing"
1112
)
@@ -87,7 +88,7 @@ func TestResolveSources(t *testing.T) {
8788
}
8889
for _, tt := range tests {
8990
t.Run(tt.name+tt.args.indexPattern, func(t *testing.T) {
90-
got, _, _ := ResolveSources(tt.args.indexPattern, &tt.args.cfg, tt.args.im)
91+
got, _, _ := ResolveSources(tt.args.indexPattern, &tt.args.cfg, tt.args.im, schema.StaticRegistry{})
9192
assert.Equalf(t, tt.want, got, "ResolveSources(%v, %v, %v)", tt.args.indexPattern, tt.args.cfg, tt.args.im)
9293
})
9394
}

‎quesma/schema/registry.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type (
2626
}
2727
TableProvider interface {
2828
TableDefinitions() map[string]Table
29+
AutodiscoveryEnabled() bool
2930
}
3031
Table struct {
3132
Columns map[string]Column
@@ -39,6 +40,14 @@ type (
3940
func (s *schemaRegistry) loadSchemas() (map[TableName]Schema, error) {
4041
definitions := s.dataSourceTableProvider.TableDefinitions()
4142
schemas := make(map[TableName]Schema)
43+
if s.dataSourceTableProvider.AutodiscoveryEnabled() {
44+
for tableName := range definitions {
45+
fields := make(map[FieldName]Field)
46+
existsInDataSource := s.populateSchemaFromTableDefinition(definitions, tableName, fields)
47+
schemas[TableName(tableName)] = NewSchema(fields, existsInDataSource)
48+
}
49+
return schemas, nil
50+
}
4251

4352
for indexName, indexConfiguration := range *s.indexConfiguration {
4453
fields := make(map[FieldName]Field)

0 commit comments

Comments
 (0)