Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 493996b

Browse files
jakozaurjakozaur
authored
Add graceful configuration error (#1464)
Currently, Quesma is in a restart loop on configuration issues, which is a terrible developer experience. Instead of being able to read the error message, it keeps restarting. My proposal instead is to return exit 0 and not restart. This allows reading messages as well as gives a much faster error in `docker compose up`.
1 parent 752cecc commit 493996b

File tree

10 files changed

+54
-75
lines changed

10 files changed

+54
-75
lines changed

‎ci/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ services:
1919
volumes:
2020
- ./quesma/logs/:/var/quesma/logs
2121
- ../examples/kibana-sample-data/quesma/config/local-dev.yaml:/mnt/ci-config.yaml
22-
restart: unless-stopped
22+
restart: on-failure
2323
log-generator:
2424
build: log-generator
2525
image: log-generator:latest

‎ci/e2e.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ services:
2323
- "9200:8080"
2424
volumes:
2525
- ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml
26-
restart: unless-stopped
26+
restart: on-failure
2727
elasticsearch:
2828
image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
2929
container_name: elasticsearch

‎cmd/experimental/main.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,10 @@ func main() {
135135
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
136136
doneCh := make(chan struct{})
137137

138-
var newConfiguration = config.LoadV2Config()
138+
var newConfiguration, configErr = config.LoadV2Config()
139+
if configErr != nil {
140+
return // We log error in LoadV2Config
141+
}
139142
var cfg = newConfiguration.TranslateToLegacyConfig()
140143

141144
if err := cfg.Validate(); err != nil {

‎cmd/experimental/v2_quesma_builder.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ import (
2222
// BuildNewQuesma creates a new quesma instance with both Ingest And Query Processors, unused yet
2323
func BuildNewQuesma() quesma_api.QuesmaBuilder {
2424

25-
var newConfiguration = config.LoadV2Config()
25+
var newConfiguration, configErr = config.LoadV2Config()
26+
if configErr != nil {
27+
os.Exit(0) // We log error in LoadV2Config, likely replace with returning an error
28+
}
2629
var cfg = newConfiguration.TranslateToLegacyConfig()
2730

2831
if err := cfg.Validate(); err != nil {

‎cmd/main.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@ func main() {
5858
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
5959
doneCh := make(chan struct{})
6060

61-
var newConfiguration = config.LoadV2Config()
61+
var newConfiguration, configErr = config.LoadV2Config()
62+
if configErr != nil {
63+
return // We log error in LoadV2Config
64+
}
6265
var cfg = newConfiguration.TranslateToLegacyConfig()
6366

6467
if err := cfg.Validate(); err != nil {

‎cmd/v2_quesma_builder.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ import (
2222
// BuildNewQuesma creates a new quesma instance with both Ingest And Query Processors, unused yet
2323
func BuildNewQuesma() quesma_api.QuesmaBuilder {
2424

25-
var newConfiguration = config.LoadV2Config()
25+
var newConfiguration, configErr = config.LoadV2Config()
26+
if configErr != nil {
27+
os.Exit(0) // We log error in LoadV2Config, likely replace with returning an error
28+
}
2629
var cfg = newConfiguration.TranslateToLegacyConfig()
2730

2831
if err := cfg.Validate(); err != nil {

‎examples/kibana-sample-data/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ services:
2121
resources:
2222
limits:
2323
memory: 512M
24-
restart: unless-stopped
24+
restart: on-failure
2525
log-generator:
2626
build: log-generator
2727
depends_on:

‎platform/config/config_v2.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,21 +152,23 @@ func (p *QuesmaProcessorConfig) IsFieldMapSyntaxEnabled(indexName string) bool {
152152
return false
153153
}
154154

155-
func LoadV2Config() QuesmaNewConfiguration {
155+
func LoadV2Config() (QuesmaNewConfiguration, error) {
156156
var v2config QuesmaNewConfiguration
157157
loadConfigFile()
158158
// We have to use custom env provider to allow array overrides
159159
if err := k.Load(Env2JsonProvider("QUESMA_", "_", nil), json.Parser(), koanf.WithMergeFunc(mergeDictFunc)); err != nil {
160-
log.Fatalf("error loading config form supplied env vars: %v", err)
160+
log.Printf("error loading config form supplied env vars: %v", err)
161+
return v2config, err
161162
}
162163
if err := k.Unmarshal("", &v2config); err != nil {
163164
log.Fatalf("error unmarshalling config: %v", err)
164165
}
165166

166167
if err := v2config.Validate(); err != nil {
167-
log.Fatalf("Config validation failed: %v", err)
168+
log.Printf("Config validation failed: %v", err)
169+
return v2config, err
168170
}
169-
return v2config
171+
return v2config, nil
170172
}
171173

172174
// validate at this level verifies the basic assumptions behind pipelines/processors/connectors,
@@ -189,7 +191,6 @@ func (c *QuesmaNewConfiguration) Validate() error {
189191
var multiErr *multierror.Error
190192
if errors.As(errAcc, &multiErr) {
191193
if len(multiErr.Errors) > 0 {
192-
log.Fatalf("Config validation failed: %v", multiErr)
193194
return multiErr
194195
}
195196
}

‎platform/config/config_v2_test.go‎

Lines changed: 27 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ import (
1111
"testing"
1212
)
1313

14+
func loadConfig(t *testing.T) QuesmaNewConfiguration {
15+
cfg, cfgErr := LoadV2Config()
16+
assert.NoError(t, cfgErr, "error loading config")
17+
if err := cfg.Validate(); err != nil {
18+
t.Fatalf("error validating config: %v", err)
19+
}
20+
return cfg
21+
}
22+
1423
func TestQuesmaConfigurationLoading(t *testing.T) {
1524

1625
os.Setenv(configFileLocationEnvVar, "./test_configs/test_config_v2.yaml")
@@ -26,11 +35,7 @@ func TestQuesmaConfigurationLoading(t *testing.T) {
2635
os.Unsetenv("QUESMA_licenseKey")
2736
os.Unsetenv("QUESMA_backendConnectors_1_config_user")
2837
})
29-
cfg := LoadV2Config()
30-
if err := cfg.Validate(); err != nil {
31-
t.Fatalf("error validating config: %v", err)
32-
}
33-
38+
cfg := loadConfig(t)
3439
legacyCfg := cfg.TranslateToLegacyConfig()
3540

3641
assert.Equal(t, licenseKeyPassedAsEnvVar, legacyCfg.LicenseKey)
@@ -74,10 +79,7 @@ func TestQuesmaConfigurationLoading(t *testing.T) {
7479

7580
func TestQuesmaTransparentProxyConfiguration(t *testing.T) {
7681
os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_as_transparent_proxy.yml")
77-
cfg := LoadV2Config()
78-
if err := cfg.Validate(); err != nil {
79-
t.Fatalf("error validating config: %v", err)
80-
}
82+
cfg := loadConfig(t)
8183
legacyConf := cfg.TranslateToLegacyConfig()
8284
assert.True(t, legacyConf.TransparentProxy)
8385
assert.Equal(t, false, legacyConf.EnableIngest)
@@ -88,10 +90,8 @@ func TestQuesmaTransparentProxyWithoutNoopConfiguration(t *testing.T) {
8890
t.Skip("not working yet")
8991

9092
os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_as_transparent_proxy_without_noop.yml")
91-
cfg := LoadV2Config()
92-
if err := cfg.Validate(); err != nil {
93-
t.Fatalf("error validating config: %v", err)
94-
}
93+
cfg := loadConfig(t)
94+
9595
legacyConf := cfg.TranslateToLegacyConfig()
9696
assert.False(t, legacyConf.TransparentProxy) // even though transparent proxy would work similarly, the user explicitly requested two Quesma pipelines
9797
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -109,10 +109,7 @@ func TestQuesmaTransparentProxyWithoutNoopConfiguration(t *testing.T) {
109109

110110
func TestQuesmaAddingHydrolixTablesToExistingElasticsearch(t *testing.T) {
111111
os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_adding_two_hydrolix_tables.yaml")
112-
cfg := LoadV2Config()
113-
if err := cfg.Validate(); err != nil {
114-
t.Fatalf("error validating config: %v", err)
115-
}
112+
cfg := loadConfig(t)
116113
legacyConf := cfg.TranslateToLegacyConfig()
117114
assert.False(t, legacyConf.TransparentProxy)
118115
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -130,10 +127,7 @@ func TestQuesmaAddingHydrolixTablesToExistingElasticsearch(t *testing.T) {
130127

131128
func TestIngestWithSingleConnector(t *testing.T) {
132129
os.Setenv(configFileLocationEnvVar, "./test_configs/ingest_with_single_connector.yaml")
133-
cfg := LoadV2Config()
134-
if err := cfg.Validate(); err != nil {
135-
t.Fatalf("error validating config: %v", err)
136-
}
130+
cfg := loadConfig(t)
137131
legacyConf := cfg.TranslateToLegacyConfig()
138132
assert.False(t, legacyConf.TransparentProxy)
139133
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -151,10 +145,7 @@ func TestIngestWithSingleConnector(t *testing.T) {
151145

152146
func TestQuesmaHydrolixQueryOnly(t *testing.T) {
153147
os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_hydrolix_tables_query_only.yaml")
154-
cfg := LoadV2Config()
155-
if err := cfg.Validate(); err != nil {
156-
t.Fatalf("error validating config: %v", err)
157-
}
148+
cfg := loadConfig(t)
158149
legacyConf := cfg.TranslateToLegacyConfig()
159150
assert.False(t, legacyConf.TransparentProxy)
160151
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -175,10 +166,7 @@ func TestQuesmaHydrolixQueryOnly(t *testing.T) {
175166

176167
func TestHasCommonTable(t *testing.T) {
177168
os.Setenv(configFileLocationEnvVar, "./test_configs/has_common_table.yaml")
178-
cfg := LoadV2Config()
179-
if err := cfg.Validate(); err != nil {
180-
t.Fatalf("error validating config: %v", err)
181-
}
169+
cfg := loadConfig(t)
182170
legacyConf := cfg.TranslateToLegacyConfig()
183171

184172
assert.Equal(t, true, legacyConf.EnableIngest)
@@ -187,7 +175,8 @@ func TestHasCommonTable(t *testing.T) {
187175

188176
func TestInvalidDualTarget(t *testing.T) {
189177
os.Setenv(configFileLocationEnvVar, "./test_configs/invalid_dual_target.yaml")
190-
cfg := LoadV2Config()
178+
cfg, cfgErr := LoadV2Config()
179+
assert.NoError(t, cfgErr, "error loading config")
191180
if err := cfg.Validate(); err != nil {
192181

193182
if !strings.Contains(err.Error(), "has invalid dual query target configuration - when you specify two targets") {
@@ -233,10 +222,7 @@ func TestMatchName(t *testing.T) {
233222

234223
func TestTargetNewVariant(t *testing.T) {
235224
os.Setenv(configFileLocationEnvVar, "./test_configs/target_new_variant.yaml")
236-
cfg := LoadV2Config()
237-
if err := cfg.Validate(); err != nil {
238-
t.Fatalf("error validating config: %v", err)
239-
}
225+
cfg := loadConfig(t)
240226
legacyConf := cfg.TranslateToLegacyConfig()
241227
assert.False(t, legacyConf.TransparentProxy)
242228
assert.Equal(t, 4, len(legacyConf.IndexConfig))
@@ -271,10 +257,7 @@ func TestTargetNewVariant(t *testing.T) {
271257

272258
func TestTargetLegacyVariant(t *testing.T) {
273259
os.Setenv(configFileLocationEnvVar, "./test_configs/target_legacy_variant.yaml")
274-
cfg := LoadV2Config()
275-
if err := cfg.Validate(); err != nil {
276-
t.Fatalf("error validating config: %v", err)
277-
}
260+
cfg := loadConfig(t)
278261
legacyConf := cfg.TranslateToLegacyConfig()
279262
assert.False(t, legacyConf.TransparentProxy)
280263
assert.Equal(t, 3, len(legacyConf.IndexConfig))
@@ -300,10 +283,7 @@ func TestTargetLegacyVariant(t *testing.T) {
300283

301284
func TestUseCommonTableGlobalProperty(t *testing.T) {
302285
os.Setenv(configFileLocationEnvVar, "./test_configs/use_common_table_global_property.yaml")
303-
cfg := LoadV2Config()
304-
if err := cfg.Validate(); err != nil {
305-
t.Fatalf("error validating config: %v", err)
306-
}
286+
cfg := loadConfig(t)
307287
legacyConf := cfg.TranslateToLegacyConfig()
308288
assert.False(t, legacyConf.TransparentProxy)
309289
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -322,10 +302,7 @@ func TestUseCommonTableGlobalProperty(t *testing.T) {
322302

323303
func TestIngestOptimizers(t *testing.T) {
324304
os.Setenv(configFileLocationEnvVar, "./test_configs/ingest_only_optimizers.yaml")
325-
cfg := LoadV2Config()
326-
if err := cfg.Validate(); err != nil {
327-
t.Fatalf("error validating config: %v", err)
328-
}
305+
cfg := loadConfig(t)
329306
legacyConf := cfg.TranslateToLegacyConfig()
330307
assert.False(t, legacyConf.TransparentProxy)
331308
assert.Equal(t, 1, len(legacyConf.IndexConfig))
@@ -343,10 +320,7 @@ func TestIngestOptimizers(t *testing.T) {
343320

344321
func TestPartitionBy(t *testing.T) {
345322
os.Setenv(configFileLocationEnvVar, "./test_configs/partition_by.yaml")
346-
cfg := LoadV2Config()
347-
if err := cfg.Validate(); err != nil {
348-
t.Fatalf("error validating config: %v", err)
349-
}
323+
cfg := loadConfig(t)
350324
legacyConf := cfg.TranslateToLegacyConfig()
351325

352326
assert.Equal(t, 2, len(legacyConf.IndexConfig))
@@ -363,10 +337,7 @@ func TestPartitionBy(t *testing.T) {
363337
func TestIndexNameRewriteRules(t *testing.T) {
364338

365339
os.Setenv(configFileLocationEnvVar, "./test_configs/index_name_rewrite_rules.yaml")
366-
cfg := LoadV2Config()
367-
if err := cfg.Validate(); err != nil {
368-
t.Fatalf("error validating config: %v", err)
369-
}
340+
cfg := loadConfig(t)
370341
legacyConf := cfg.TranslateToLegacyConfig()
371342

372343
assert.Equal(t, 4, len(legacyConf.IndexNameRewriteRules))
@@ -383,10 +354,7 @@ func TestIndexNameRewriteRules(t *testing.T) {
383354

384355
func TestStringColumnIsTextDefaultBehavior(t *testing.T) {
385356
os.Setenv(configFileLocationEnvVar, "./test_configs/partition_by.yaml")
386-
cfg := LoadV2Config()
387-
if err := cfg.Validate(); err != nil {
388-
t.Fatalf("error validating config: %v", err)
389-
}
357+
cfg := loadConfig(t)
390358
legacyConf := cfg.TranslateToLegacyConfig()
391359

392360
assert.Equal(t, "text", legacyConf.DefaultStringColumnType)
@@ -395,10 +363,7 @@ func TestStringColumnIsTextDefaultBehavior(t *testing.T) {
395363

396364
func TestStringColumnIsKeyword(t *testing.T) {
397365
os.Setenv(configFileLocationEnvVar, "./test_configs/string_column_is_keyword_field.yaml")
398-
cfg := LoadV2Config()
399-
if err := cfg.Validate(); err != nil {
400-
t.Fatalf("error validating config: %v", err)
401-
}
366+
cfg := loadConfig(t)
402367
legacyConf := cfg.TranslateToLegacyConfig()
403368

404369
assert.Equal(t, "keyword", legacyConf.DefaultStringColumnType)

‎platform/config/env2json_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ func TestEnv2Json_arraysByName(t *testing.T) {
4141
os.Unsetenv("QUESMA_backendConnectors_my-clickhouse-data-source_config_password")
4242
})
4343

44-
cfg := LoadV2Config()
44+
cfg, err := LoadV2Config()
45+
assert.NoError(t, err)
4546
assert.Len(t, cfg.BackendConnectors, 2)
4647
clickHouseBackend := cfg.BackendConnectors[1]
4748
assert.Equal(t, "my-clickhouse-data-source", clickHouseBackend.Name)

0 commit comments

Comments
 (0)