Configuration clean up (#823)

`QuesmaConfiguration` is a configuration intermediate representation
now. We can remove `koanf` annotations here to avoid any confusion.

Author: nablaone

quesma/quesma/config/config.go

@@ -6,10 +6,8 @@ import (
 	"fmt"
 	"github.com/hashicorp/go-multierror"
 	"github.com/knadh/koanf/parsers/yaml"
-	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/v2"
-	"github.com/rs/zerolog"
 	"log"
 	"os"
 	"quesma/elasticsearch/elasticsearch_field_types"
@@ -29,58 +27,27 @@ var (
 
 type QuesmaConfiguration struct {
 	// both clickhouse and hydrolix connections are going to be deprecated and everything is going to live under connector
-	Connectors       map[string]RelationalDbConfiguration `koanf:"connectors"`
-	TransparentProxy bool                                 `koanf:"transparentProxy"`
-	InstallationId   string                               `koanf:"installationId"`
-	LicenseKey       string                               `koanf:"licenseKey"`
+	Connectors       map[string]RelationalDbConfiguration
+	TransparentProxy bool
+	InstallationId   string
+	LicenseKey       string
 	//deprecated
-	ClickHouse RelationalDbConfiguration `koanf:"clickhouse"`
+	ClickHouse RelationalDbConfiguration
 	//deprecated
-	Hydrolix                   RelationalDbConfiguration     `koanf:"hydrolix"`
-	Elasticsearch              ElasticsearchConfiguration    `koanf:"elasticsearch"`
-	IndexConfig                map[string]IndexConfiguration `koanf:"indexes"`
-	Logging                    LoggingConfiguration          `koanf:"logging"`
-	PublicTcpPort              network.Port                  `koanf:"port"`
-	IngestStatistics           bool                          `koanf:"ingestStatistics"`
-	QuesmaInternalTelemetryUrl *Url                          `koanf:"internalTelemetryUrl"`
-	DisableAuth                bool                          `koanf:"disableAuth"`
+	Hydrolix                   RelationalDbConfiguration
+	Elasticsearch              ElasticsearchConfiguration
+	IndexConfig                map[string]IndexConfiguration
+	Logging                    LoggingConfiguration
+	PublicTcpPort              network.Port
+	IngestStatistics           bool
+	QuesmaInternalTelemetryUrl *Url
+	DisableAuth                bool
 	AutodiscoveryEnabled       bool
 
 	EnableIngest      bool // this is computed from the configuration 2.0
 	CreateCommonTable bool
 }
 
-type LoggingConfiguration struct {
-	Path              string         `koanf:"path"`
-	Level             *zerolog.Level `koanf:"level"`
-	RemoteLogDrainUrl *Url           `koanf:"remoteUrl"`
-	FileLogging       bool           `koanf:"fileLogging"`
-}
-
-type RelationalDbConfiguration struct {
-	//ConnectorName string `koanf:"name"`
-	ConnectorType string `koanf:"type"`
-	Url           *Url   `koanf:"url"`
-	User          string `koanf:"user"`
-	Password      string `koanf:"password"`
-	Database      string `koanf:"database"`
-	AdminUrl      *Url   `koanf:"adminUrl"`
-	DisableTLS    bool   `koanf:"disableTLS"`
-}
-
-type OptimizerConfiguration struct {
-	Disabled   bool              `koanf:"disabled"`
-	Properties map[string]string `koanf:"properties"`
-}
-
-func (c *RelationalDbConfiguration) IsEmpty() bool {
-	return c != nil && c.Url == nil && c.User == "" && c.Password == "" && c.Database == ""
-}
-
-func (c *RelationalDbConfiguration) IsNonEmpty() bool {
-	return !c.IsEmpty()
-}
-
 func (c *QuesmaConfiguration) AliasFields(indexName string) map[string]string {
 	aliases := make(map[string]string)
 	if indexConfig, found := c.IndexConfig[indexName]; found {
@@ -99,34 +66,6 @@ func MatchName(pattern, name string) bool {
 
 var k = koanf.New(".")
 
-func Load() QuesmaConfiguration {
-	var config QuesmaConfiguration
-
-	loadConfigFile()
-	if err := k.Load(env.Provider("QUESMA_", ".", func(s string) string {
-		// This enables overriding config values with environment variables. It's case-sensitive, just like the YAML.
-		// Examples:
-		// `QUESMA_logging_level=debug` overrides `logging.level` in the config file
-		// `QUESMA_licenseKey=arbitrary-license-key` overrides `licenseKey` in the config file
-		return strings.Replace(strings.TrimPrefix(s, "QUESMA_"), "_", ".", -1)
-	}), nil); err != nil {
-		log.Fatalf("error loading config form supplied env vars: %v", err)
-	}
-	if err := k.Unmarshal("", &config); err != nil {
-		log.Fatalf("error unmarshalling config: %v", err)
-	}
-	for name, idxConfig := range config.IndexConfig {
-		idxConfig.Name = name
-		config.IndexConfig[name] = idxConfig
-		if idxConfig.SchemaOverrides != nil {
-			for fieldName, configuration := range idxConfig.SchemaOverrides.Fields {
-				idxConfig.SchemaOverrides.Fields[fieldName] = configuration
-			}
-		}
-	}
-	return config
-}
-
 func loadConfigFile() {
 	var configPath string
 	if configFileName, isSet := os.LookupEnv(configFileLocationEnvVar); isSet {

quesma/quesma/config/config_v2.go

@@ -51,6 +51,13 @@ type QuesmaNewConfiguration struct {
 	DisableTelemetry           bool                 `koanf:"disableTelemetry"`
 }
 
+type LoggingConfiguration struct {
+	Path              string         `koanf:"path"`
+	Level             *zerolog.Level `koanf:"level"`
+	RemoteLogDrainUrl *Url           `koanf:"remoteUrl"`
+	FileLogging       bool           `koanf:"fileLogging"`
+}
+
 type Pipeline struct {
 	Name               string   `koanf:"name"`
 	FrontendConnectors []string `koanf:"frontendConnectors"`
@@ -75,6 +82,25 @@ type BackendConnector struct {
 	Config RelationalDbConfiguration `koanf:"config"`
 }
 
+type RelationalDbConfiguration struct {
+	//ConnectorName string `koanf:"name"`
+	ConnectorType string `koanf:"type"`
+	Url           *Url   `koanf:"url"`
+	User          string `koanf:"user"`
+	Password      string `koanf:"password"`
+	Database      string `koanf:"database"`
+	AdminUrl      *Url   `koanf:"adminUrl"`
+	DisableTLS    bool   `koanf:"disableTLS"`
+}
+
+func (c *RelationalDbConfiguration) IsEmpty() bool {
+	return c != nil && c.Url == nil && c.User == "" && c.Password == "" && c.Database == ""
+}
+
+func (c *RelationalDbConfiguration) IsNonEmpty() bool {
+	return !c.IsEmpty()
+}
+
 type Processor struct {
 	Name   string                `koanf:"name"`
 	Type   ProcessorType         `koanf:"type"`

quesma/quesma/config/index_config.go

@@ -26,6 +26,11 @@ type IndexConfiguration struct {
 	IngestTarget []string
 }
 
+type OptimizerConfiguration struct {
+	Disabled   bool              `koanf:"disabled"`
+	Properties map[string]string `koanf:"properties"`
+}
+
 func (c IndexConfiguration) String() string {
 	var builder strings.Builder