Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 5d19d94

Browse files
pdelewskipdelewski
authored
Target config extended configuration (#945)
This PR implements new variant of `target` configuration where it's a list not an array and can have more attributes (potentially in the future) Current variant is still supported and is a fallback in the case extended version is missing Below ``` kibana_sample_data_flights: target: - my-clickhouse-data-source: useCommonTable: true ``` is equivalent of ``` kibana_sample_data_flights: target: [my-clickhouse-data-source] useCommonTable: true ``` PR contains a bit of duplicated code, something that I would prefer fix in next PR.
1 parent c6ed018 commit 5d19d94

File tree

4 files changed

+315
-44
lines changed

4 files changed

+315
-44
lines changed

‎quesma/quesma/config/config_v2.go‎

Lines changed: 215 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -377,18 +377,37 @@ func (c *QuesmaNewConfiguration) validateProcessor(p Processor) error {
377377
return fmt.Errorf("index name '%s' in processor configuration is an index pattern, not allowed", indexName)
378378
}
379379
if p.Type == QuesmaV1ProcessorQuery {
380-
if len(indexConfig.Target) > 2 {
381-
return fmt.Errorf("configuration of index %s must have at most two targets (query processor)", indexName)
380+
if _, ok := indexConfig.Target.([]interface{}); ok {
381+
if len(indexConfig.Target.([]interface{})) > 2 {
382+
return fmt.Errorf("configuration of index %s must have at most two targets (query processor)", indexName)
383+
}
382384
}
383385
} else {
384-
if len(indexConfig.Target) > 2 {
385-
return fmt.Errorf("configuration of index %s must have at most two targets (ingest processor)", indexName)
386+
if _, ok := indexConfig.Target.([]interface{}); ok {
387+
if len(indexConfig.Target.([]interface{})) > 2 {
388+
return fmt.Errorf("configuration of index %s must have at most two targets (ingest processor)", indexName)
389+
}
386390
}
387391
}
388-
389-
for _, target := range indexConfig.Target {
390-
if c.getBackendConnectorByName(target) == nil {
391-
return fmt.Errorf("invalid target %s in configuration of index %s", target, indexName)
392+
targets, errTarget := c.getTargetsExtendedConfig(indexConfig.Target)
393+
if errTarget != nil {
394+
return errTarget
395+
}
396+
// fallback to old style, simplified target configuration
397+
if len(targets) > 0 {
398+
for _, target := range targets {
399+
if c.getBackendConnectorByName(target.target) == nil {
400+
return fmt.Errorf("invalid target %s in configuration of index %s", target, indexName)
401+
}
402+
}
403+
}
404+
if len(targets) == 0 {
405+
if _, ok := indexConfig.Target.([]interface{}); ok {
406+
for _, target := range indexConfig.Target.([]interface{}) {
407+
if c.getBackendConnectorByName(target.(string)) == nil {
408+
return fmt.Errorf("invalid target %s in configuration of index %s", target, indexName)
409+
}
410+
}
392411
}
393412
}
394413
}
@@ -545,13 +564,36 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
545564

546565
// Handle default index configuration
547566
defaultConfig := queryProcessor.Config.IndexConfig[DefaultWildcardIndexName]
548-
for _, target := range defaultConfig.Target {
549-
if targetType, found := c.getTargetType(target); found {
550-
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
551-
} else {
552-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
567+
targets, errTarget := c.getTargetsExtendedConfig(defaultConfig.Target)
568+
if errTarget != nil {
569+
errAcc = multierror.Append(errAcc, errTarget)
570+
}
571+
if len(targets) > 0 {
572+
for _, target := range targets {
573+
if targetType, found := c.getTargetType(target.target); found {
574+
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
575+
} else {
576+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
577+
}
578+
if val, exists := target.properties["useCommonTable"]; exists {
579+
conf.CreateCommonTable = val == "true"
580+
conf.UseCommonTableForWildcard = val == "true"
581+
}
553582
}
554583
}
584+
// fallback to old style, simplified target configuration
585+
if len(targets) == 0 {
586+
if _, ok := defaultConfig.Target.([]interface{}); ok {
587+
for _, target := range defaultConfig.Target.([]interface{}) {
588+
if targetType, found := c.getTargetType(target.(string)); found {
589+
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
590+
} else {
591+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
592+
}
593+
}
594+
}
595+
}
596+
555597
if defaultConfig.UseCommonTable {
556598
// We set both flags to true here
557599
// as creating common table depends on the first one
@@ -569,15 +611,34 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
569611
for indexName, indexConfig := range queryProcessor.Config.IndexConfig {
570612
processedConfig := indexConfig
571613
processedConfig.Name = indexName
572-
573-
for _, target := range indexConfig.Target {
574-
if targetType, found := c.getTargetType(target); found {
575-
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
576-
} else {
577-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
614+
targets, errTarget := c.getTargetsExtendedConfig(indexConfig.Target)
615+
if errTarget != nil {
616+
errAcc = multierror.Append(errAcc, errTarget)
617+
}
618+
if len(targets) > 0 {
619+
for _, target := range targets {
620+
if targetType, found := c.getTargetType(target.target); found {
621+
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
622+
} else {
623+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
624+
}
625+
if val, exists := target.properties["useCommonTable"]; exists {
626+
processedConfig.UseCommonTable = val == "true"
627+
}
628+
}
629+
}
630+
// fallback to old style, simplified target configuration
631+
if len(targets) == 0 {
632+
if _, ok := indexConfig.Target.([]interface{}); ok {
633+
for _, target := range indexConfig.Target.([]interface{}) {
634+
if targetType, found := c.getTargetType(target.(string)); found {
635+
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
636+
} else {
637+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
638+
}
639+
}
578640
}
579641
}
580-
581642
if len(processedConfig.QueryTarget) == 2 && !((processedConfig.QueryTarget[0] == ClickhouseTarget && processedConfig.QueryTarget[1] == ElasticsearchTarget) ||
582643
(processedConfig.QueryTarget[0] == ElasticsearchTarget && processedConfig.QueryTarget[1] == ClickhouseTarget)) {
583644
errAcc = multierror.Append(errAcc, fmt.Errorf("index %s has invalid dual query target configuration", indexName))
@@ -619,11 +680,33 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
619680

620681
// Handle default index configuration
621682
defaultConfig := queryProcessor.Config.IndexConfig[DefaultWildcardIndexName]
622-
for _, target := range defaultConfig.Target {
623-
if targetType, found := c.getTargetType(target); found {
624-
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
625-
} else {
626-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
683+
targets, errTarget := c.getTargetsExtendedConfig(defaultConfig.Target)
684+
if errTarget != nil {
685+
errAcc = multierror.Append(errAcc, errTarget)
686+
}
687+
if len(targets) > 0 {
688+
for _, target := range targets {
689+
if targetType, found := c.getTargetType(target.target); found {
690+
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
691+
} else {
692+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
693+
}
694+
if val, exists := target.properties["useCommonTable"]; exists {
695+
conf.CreateCommonTable = val == "true"
696+
conf.UseCommonTableForWildcard = val == "true"
697+
}
698+
}
699+
}
700+
// fallback to old style, simplified target configuration
701+
if len(targets) == 0 {
702+
if _, ok := defaultConfig.Target.([]interface{}); ok {
703+
for _, target := range defaultConfig.Target.([]interface{}) {
704+
if targetType, found := c.getTargetType(target.(string)); found {
705+
defaultConfig.QueryTarget = append(defaultConfig.QueryTarget, targetType)
706+
} else {
707+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
708+
}
709+
}
627710
}
628711
}
629712
if defaultConfig.UseCommonTable {
@@ -634,11 +717,33 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
634717
}
635718

636719
ingestProcessorDefaultIndexConfig := ingestProcessor.Config.IndexConfig[DefaultWildcardIndexName]
637-
for _, target := range ingestProcessor.Config.IndexConfig[DefaultWildcardIndexName].Target {
638-
if targetType, found := c.getTargetType(target); found {
639-
defaultConfig.IngestTarget = append(defaultConfig.IngestTarget, targetType)
640-
} else {
641-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
720+
targets, errTarget = c.getTargetsExtendedConfig(ingestProcessorDefaultIndexConfig.Target)
721+
if errTarget != nil {
722+
errAcc = multierror.Append(errAcc, errTarget)
723+
}
724+
if len(targets) > 0 {
725+
for _, target := range targets {
726+
if targetType, found := c.getTargetType(target.target); found {
727+
defaultConfig.IngestTarget = append(defaultConfig.IngestTarget, targetType)
728+
} else {
729+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
730+
}
731+
if val, exists := target.properties["useCommonTable"]; exists {
732+
conf.CreateCommonTable = val == "true"
733+
conf.UseCommonTableForWildcard = val == "true"
734+
}
735+
}
736+
}
737+
// fallback to old style, simplified target configuration
738+
if len(targets) == 0 {
739+
if _, ok := ingestProcessor.Config.IndexConfig[DefaultWildcardIndexName].Target.([]interface{}); ok {
740+
for _, target := range ingestProcessor.Config.IndexConfig[DefaultWildcardIndexName].Target.([]interface{}) {
741+
if targetType, found := c.getTargetType(target.(string)); found {
742+
defaultConfig.IngestTarget = append(defaultConfig.IngestTarget, targetType)
743+
} else {
744+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of %s", target, DefaultWildcardIndexName))
745+
}
746+
}
642747
}
643748
}
644749
if ingestProcessorDefaultIndexConfig.UseCommonTable {
@@ -668,15 +773,34 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
668773
processedConfig.Name = indexName
669774

670775
processedConfig.IngestTarget = defaultConfig.IngestTarget
671-
672-
for _, target := range indexConfig.Target {
673-
if targetType, found := c.getTargetType(target); found {
674-
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
675-
} else {
676-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
776+
targets, errTarget = c.getTargetsExtendedConfig(indexConfig.Target)
777+
if errTarget != nil {
778+
errAcc = multierror.Append(errAcc, errTarget)
779+
}
780+
if len(targets) > 0 {
781+
for _, target := range targets {
782+
if targetType, found := c.getTargetType(target.target); found {
783+
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
784+
} else {
785+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
786+
}
787+
if val, exists := target.properties["useCommonTable"]; exists {
788+
processedConfig.UseCommonTable = val == true
789+
}
790+
}
791+
}
792+
// fallback to old style, simplified target configuration
793+
if len(targets) == 0 {
794+
if _, ok := indexConfig.Target.([]interface{}); ok {
795+
for _, target := range indexConfig.Target.([]interface{}) {
796+
if targetType, found := c.getTargetType(target.(string)); found {
797+
processedConfig.QueryTarget = append(processedConfig.QueryTarget, targetType)
798+
} else {
799+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
800+
}
801+
}
677802
}
678803
}
679-
680804
if len(processedConfig.QueryTarget) == 2 && !((processedConfig.QueryTarget[0] == ClickhouseTarget && processedConfig.QueryTarget[1] == ElasticsearchTarget) ||
681805
(processedConfig.QueryTarget[0] == ElasticsearchTarget && processedConfig.QueryTarget[1] == ClickhouseTarget)) {
682806
errAcc = multierror.Append(errAcc, fmt.Errorf("index %s has invalid dual query target configuration", indexName))
@@ -709,14 +833,34 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
709833
}
710834

711835
processedConfig.IngestTarget = make([]string, 0) // reset previously set defaultConfig.IngestTarget
712-
for _, target := range indexConfig.Target {
713-
if targetType, found := c.getTargetType(target); found {
714-
processedConfig.IngestTarget = append(processedConfig.IngestTarget, targetType)
715-
} else {
716-
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
836+
targets, errTarget = c.getTargetsExtendedConfig(indexConfig.Target)
837+
if errTarget != nil {
838+
errAcc = multierror.Append(errAcc, errTarget)
839+
}
840+
if len(targets) > 0 {
841+
for _, target := range targets {
842+
if targetType, found := c.getTargetType(target.target); found {
843+
processedConfig.IngestTarget = append(processedConfig.IngestTarget, targetType)
844+
} else {
845+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
846+
}
847+
if val, exists := target.properties["useCommonTable"]; exists {
848+
processedConfig.UseCommonTable = val == true
849+
}
850+
}
851+
}
852+
// fallback to old style, simplified target configuration
853+
if len(targets) == 0 {
854+
if _, ok := indexConfig.Target.([]interface{}); ok {
855+
for _, target := range indexConfig.Target.([]interface{}) {
856+
if targetType, found := c.getTargetType(target.(string)); found {
857+
processedConfig.IngestTarget = append(processedConfig.IngestTarget, targetType)
858+
} else {
859+
errAcc = multierror.Append(errAcc, fmt.Errorf("invalid target %s in configuration of index %s", target, indexName))
860+
}
861+
}
717862
}
718863
}
719-
720864
conf.IndexConfig[indexName] = processedConfig
721865
}
722866
}
@@ -824,3 +968,31 @@ func (c *QuesmaNewConfiguration) validateBackendConnectors() error {
824968
func getAllowedProcessorTypes() []ProcessorType {
825969
return []ProcessorType{QuesmaV1ProcessorNoOp, QuesmaV1ProcessorQuery, QuesmaV1ProcessorIngest}
826970
}
971+
972+
func (c *QuesmaNewConfiguration) getTargetsExtendedConfig(target any) ([]struct {
973+
target string
974+
properties map[string]interface{}
975+
}, error) {
976+
result := make([]struct {
977+
target string
978+
properties map[string]interface{}
979+
}, 0)
980+
981+
if targets, ok := target.([]interface{}); ok {
982+
for _, target := range targets {
983+
if targetMap, ok := target.(map[string]interface{}); ok {
984+
for name, settings := range targetMap {
985+
if settingsMap, ok := settings.(map[string]interface{}); ok {
986+
result = append(result, struct {
987+
target string
988+
properties map[string]interface{}
989+
}{target: name, properties: settingsMap})
990+
} else {
991+
return nil, fmt.Errorf("invalid target properties for target %s", name)
992+
}
993+
}
994+
}
995+
}
996+
}
997+
return result, nil
998+
}

‎quesma/quesma/config/config_v2_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,3 +229,31 @@ func TestMatchName(t *testing.T) {
229229
})
230230
}
231231
}
232+
233+
func TestTargetNewVariant(t *testing.T) {
234+
os.Setenv(configFileLocationEnvVar, "./test_configs/target_new_variant.yaml")
235+
cfg := LoadV2Config()
236+
if err := cfg.Validate(); err != nil {
237+
t.Fatalf("error validating config: %v", err)
238+
}
239+
legacyConf := cfg.TranslateToLegacyConfig()
240+
assert.False(t, legacyConf.TransparentProxy)
241+
assert.Equal(t, 3, len(legacyConf.IndexConfig))
242+
ecommerce := legacyConf.IndexConfig["kibana_sample_data_ecommerce"]
243+
flights := legacyConf.IndexConfig["kibana_sample_data_flights"]
244+
logs := legacyConf.IndexConfig["kibana_sample_data_logs"]
245+
246+
assert.Equal(t, []string{ClickhouseTarget}, ecommerce.QueryTarget)
247+
assert.Equal(t, []string{ClickhouseTarget}, ecommerce.IngestTarget)
248+
249+
assert.Equal(t, []string{ClickhouseTarget}, flights.QueryTarget)
250+
assert.Equal(t, []string{ClickhouseTarget}, flights.IngestTarget)
251+
252+
assert.Equal(t, []string{ClickhouseTarget}, logs.QueryTarget)
253+
assert.Equal(t, []string{ClickhouseTarget}, logs.IngestTarget)
254+
255+
assert.Equal(t, false, flights.UseCommonTable)
256+
assert.Equal(t, false, ecommerce.UseCommonTable)
257+
assert.Equal(t, true, logs.UseCommonTable)
258+
assert.Equal(t, true, legacyConf.EnableIngest)
259+
}

‎quesma/quesma/config/index_config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type IndexConfiguration struct {
1818
Optimizers map[string]OptimizerConfiguration `koanf:"optimizers"`
1919
Override string `koanf:"override"`
2020
UseCommonTable bool `koanf:"useCommonTable"`
21-
Target []string `koanf:"target"`
21+
Target any `koanf:"target"`
2222

2323
// Computed based on the overall configuration
2424
Name string

0 commit comments

Comments
 (0)