Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 06327f2

Browse files
nablaonenablaone
authored
Create common table if needed (#828)
We should create the common table if there is an index using it. In other cases, it emits unnecessary logs. It fixes a bug while processing the config.
1 parent 4fe5a7c commit 06327f2

File tree

6 files changed

+170
-6
lines changed

6 files changed

+170
-6
lines changed

‎quesma/main.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,10 @@ func main() {
9494
var ingestProcessor *ingest.IngestProcessor
9595

9696
if cfg.EnableIngest {
97-
// Ensure common table exists. This table have to be created before ingest processor starts
98-
common_table.EnsureCommonTableExists(connectionPool)
97+
if cfg.CreateCommonTable {
98+
// Ensure common table exists. This table have to be created before ingest processor starts
99+
common_table.EnsureCommonTableExists(connectionPool)
100+
}
99101

100102
ingestProcessor = ingest.NewEmptyIngestProcessor(&cfg, connectionPool, phoneHomeAgent, tableDisco, schemaRegistry, virtualTableStorage)
101103
} else {

‎quesma/quesma/config/config.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ type QuesmaConfiguration struct {
4646
DisableAuth bool `koanf:"disableAuth"`
4747
AutodiscoveryEnabled bool
4848

49-
EnableIngest bool // this is computed from the configuration 2.0
49+
EnableIngest bool // this is computed from the configuration 2.0
50+
CreateCommonTable bool
5051
}
5152

5253
type LoggingConfiguration struct {

‎quesma/quesma/config/config_v2.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -517,10 +517,11 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
517517
processedConfig.QueryTarget = append(processedConfig.QueryTarget, ClickhouseTarget)
518518
}
519519

520-
if len(processedConfig.QueryTarget) == 2 && !(processedConfig.QueryTarget[0] == ClickhouseTarget && processedConfig.QueryTarget[1] == ElasticsearchTarget) {
520+
if len(processedConfig.QueryTarget) == 2 && !(indexConfig.Target[0] == relationalDBBackendName && indexConfig.Target[1] == elasticBackendName) {
521521
errAcc = multierror.Append(errAcc, fmt.Errorf("index %s has invalid dual query target configuration - when you specify two targets, ClickHouse has to be the primary one and Elastic has to be the secondary one", indexName))
522522
continue
523523
}
524+
524525
if len(processedConfig.QueryTarget) == 2 {
525526
// Turn on A/B testing
526527
processedConfig.Optimizers = make(map[string]OptimizerConfiguration)
@@ -579,7 +580,7 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
579580
processedConfig.QueryTarget = append(processedConfig.QueryTarget, ClickhouseTarget)
580581
}
581582

582-
if len(processedConfig.QueryTarget) == 2 && !(processedConfig.QueryTarget[0] == ClickhouseTarget && processedConfig.QueryTarget[1] == ElasticsearchTarget) {
583+
if len(processedConfig.QueryTarget) == 2 && !(indexConfig.Target[0] == relationalDBBackendName && indexConfig.Target[1] == elasticBackendName) {
583584
errAcc = multierror.Append(errAcc, fmt.Errorf("index %s has invalid dual query target configuration - when you specify two targets, ClickHouse has to be the primary one and Elastic has to be the secondary one", indexName))
584585
continue
585586
}
@@ -632,6 +633,13 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
632633

633634
END:
634635

636+
for _, idxCfg := range conf.IndexConfig {
637+
if idxCfg.UseCommonTable {
638+
conf.CreateCommonTable = true
639+
break
640+
}
641+
}
642+
635643
if !conf.TransparentProxy {
636644
if relationalDBErr != nil {
637645
errAcc = multierror.Append(errAcc, relationalDBErr)

‎quesma/quesma/config/config_v2_test.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"github.com/stretchr/testify/assert"
88
"os"
9+
"strings"
910
"testing"
1011
)
1112

@@ -70,6 +71,8 @@ func TestQuesmaTransparentProxyConfiguration(t *testing.T) {
7071
}
7172
legacyConf := cfg.TranslateToLegacyConfig()
7273
assert.True(t, legacyConf.TransparentProxy)
74+
assert.Equal(t, false, legacyConf.EnableIngest)
75+
assert.Equal(t, false, legacyConf.CreateCommonTable)
7376
}
7477

7578
func TestQuesmaAddingHydrolixTablesToExistingElasticsearch(t *testing.T) {
@@ -89,7 +92,8 @@ func TestQuesmaAddingHydrolixTablesToExistingElasticsearch(t *testing.T) {
8992

9093
assert.Equal(t, []string{"clickhouse"}, logsIndexConf.QueryTarget)
9194
assert.Equal(t, []string{"elasticsearch"}, logsIndexConf.IngestTarget)
92-
95+
assert.Equal(t, true, legacyConf.EnableIngest)
96+
assert.Equal(t, false, legacyConf.CreateCommonTable)
9397
}
9498

9599
func TestQuesmaHydrolixQueryOnly(t *testing.T) {
@@ -113,6 +117,32 @@ func TestQuesmaHydrolixQueryOnly(t *testing.T) {
113117

114118
assert.Equal(t, false, legacyConf.EnableIngest)
115119
assert.Equal(t, false, legacyConf.IngestStatistics)
120+
assert.Equal(t, false, legacyConf.CreateCommonTable)
121+
}
122+
123+
func TestHasCommonTable(t *testing.T) {
124+
os.Setenv(configFileLocationEnvVar, "./test_configs/has_common_table.yaml")
125+
cfg := LoadV2Config()
126+
if err := cfg.Validate(); err != nil {
127+
t.Fatalf("error validating config: %v", err)
128+
}
129+
legacyConf := cfg.TranslateToLegacyConfig()
130+
131+
assert.Equal(t, true, legacyConf.EnableIngest)
132+
assert.Equal(t, true, legacyConf.CreateCommonTable)
133+
}
134+
135+
func TestInvalidDualTarget(t *testing.T) {
136+
os.Setenv(configFileLocationEnvVar, "./test_configs/invalid_dual_target.yaml")
137+
cfg := LoadV2Config()
138+
if err := cfg.Validate(); err != nil {
139+
140+
if !strings.Contains(err.Error(), "has invalid dual query target configuration - when you specify two targets") {
141+
t.Fatalf("unexpected error: %v", err)
142+
}
143+
144+
t.Fatalf("expected error, but got none")
145+
}
116146
}
117147

118148
func TestMatchName(t *testing.T) {

‎quesma/quesma/config/test_configs/has_common_table.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
installationId: #HYDROLIX_REQUIRES_THIS
2+
frontendConnectors:
3+
- name: elastic-ingest
4+
type: elasticsearch-fe-ingest
5+
config:
6+
listenPort: 8080
7+
- name: elastic-query
8+
type: elasticsearch-fe-query
9+
config:
10+
listenPort: 8080
11+
backendConnectors:
12+
- name: E
13+
type: elasticsearch
14+
config:
15+
url: "http://elasticsearch:9200"
16+
user: elastic
17+
password: quesmaquesma
18+
- name: C
19+
type: clickhouse-os
20+
config:
21+
url: "clickhouse://clickhouse:9000"
22+
ingestStatistics: true
23+
processors:
24+
- name: QP
25+
type: quesma-v1-processor-query
26+
config:
27+
indexes:
28+
logs-1:
29+
target: [ E ]
30+
logs-2:
31+
target: [ E ]
32+
logs-3:
33+
target: [ C, E ]
34+
logs-4:
35+
useCommonTable: true
36+
target: [ C ]
37+
logs-5:
38+
useCommonTable: true
39+
target: [ C ]
40+
"*":
41+
target: [ E ]
42+
43+
- name: IP
44+
type: quesma-v1-processor-ingest
45+
config:
46+
indexes:
47+
logs-1:
48+
target: [ E ]
49+
logs-2:
50+
target: [ E ]
51+
logs-3:
52+
target: [ C, E ]
53+
logs-4:
54+
useCommonTable: true
55+
target: [ C ]
56+
"*":
57+
target: [ E ]
58+
logs-5:
59+
useCommonTable: true
60+
target: [ ]
61+
62+
pipelines:
63+
- name: my-elasticsearch-proxy-read
64+
frontendConnectors: [ elastic-query ]
65+
processors: [ QP ]
66+
backendConnectors: [ E, C ]
67+
- name: my-elasticsearch-proxy-write
68+
frontendConnectors: [ elastic-ingest ]
69+
processors: [ IP ]
70+
backendConnectors: [ E, C ]

‎quesma/quesma/config/test_configs/invalid_dual_target.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
installationId: #HYDROLIX_REQUIRES_THIS
2+
frontendConnectors:
3+
- name: elastic-ingest
4+
type: elasticsearch-fe-ingest
5+
config:
6+
listenPort: 8080
7+
- name: elastic-query
8+
type: elasticsearch-fe-query
9+
config:
10+
listenPort: 8080
11+
backendConnectors:
12+
- name: E
13+
type: elasticsearch
14+
config:
15+
url: "http://elasticsearch:9200"
16+
user: elastic
17+
password: quesmaquesma
18+
- name: C
19+
type: clickhouse-os
20+
config:
21+
url: "clickhouse://clickhouse:9000"
22+
ingestStatistics: true
23+
processors:
24+
- name: QP
25+
type: quesma-v1-processor-query
26+
config:
27+
indexes:
28+
invalid:
29+
target: [ E,C ]
30+
"*":
31+
target: [ E ]
32+
33+
- name: IP
34+
type: quesma-v1-processor-ingest
35+
config:
36+
indexes:
37+
invalid:
38+
target: [ C, E ]
39+
"*":
40+
target: [ E ]
41+
logs-5:
42+
useCommonTable: true
43+
target: [ ]
44+
45+
pipelines:
46+
- name: my-elasticsearch-proxy-read
47+
frontendConnectors: [ elastic-query ]
48+
processors: [ QP ]
49+
backendConnectors: [ E, C ]
50+
- name: my-elasticsearch-proxy-write
51+
frontendConnectors: [ elastic-ingest ]
52+
processors: [ IP ]
53+
backendConnectors: [ E, C ]

0 commit comments

Comments
 (0)