Old admin console in v2 api (#1224)

Another round of v2 efforts. 

This time the only "invasive" part is commenting out one validation
which is incompatible with v2 realm.

---------

Signed-off-by: Przemysław Hejman <[email protected]>

Author: mieciu

quesma/backend_connectors/basic_sql_backend_connector.go

@@ -6,6 +6,7 @@ package backend_connectors
 import (
 	"context"
 	"database/sql"
+	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 )
 
@@ -15,6 +16,7 @@ type SqlBackendConnector interface {
 
 type BasicSqlBackendConnector struct {
 	connection *sql.DB
+	cfg        *config.RelationalDbConfiguration
 }
 
 type SqlRows struct {
@@ -37,6 +39,15 @@ func (p *SqlRows) Err() error {
 	return p.rows.Err()
 }
 
+func (p *BasicSqlBackendConnector) Open() error {
+	conn, err := initDBConnection(p.cfg)
+	if err != nil {
+		return err
+	}
+	p.connection = conn
+	return nil
+}
+
 func (p *BasicSqlBackendConnector) GetDB() *sql.DB {
 	return p.connection
 }

quesma/backend_connectors/clickhouse_backend_connector.go

@@ -4,23 +4,26 @@
 package backend_connectors
 
 import (
+	"crypto/tls"
 	"database/sql"
 	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/QuesmaOrg/quesma/quesma/buildinfo"
+	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 
 	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 )
 
 type ClickHouseBackendConnector struct {
 	BasicSqlBackendConnector
-	Endpoint string
+	cfg *config.RelationalDbConfiguration
 }
 
 func (p *ClickHouseBackendConnector) GetId() quesma_api.BackendConnectorType {
 	return quesma_api.ClickHouseSQLBackend
 }
 
 func (p *ClickHouseBackendConnector) Open() error {
-	conn, err := initDBConnection()
+	conn, err := initDBConnection(p.cfg)
 	if err != nil {
 		return err
 	}
@@ -29,34 +32,46 @@ func (p *ClickHouseBackendConnector) Open() error {
 }
 
 // func initDBConnection(c *config.QuesmaConfiguration, tlsConfig *tls.Config) *sql.DB {
-func initDBConnection() (*sql.DB, error) {
-	options := clickhouse.Options{Addr: []string{"localhost:9000"}}
+func initDBConnection(c *config.RelationalDbConfiguration) (*sql.DB, error) {
+	options := clickhouse.Options{Addr: []string{c.Url.Host}}
+	if c.User != "" || c.Password != "" || c.Database != "" {
+
+		options.Auth = clickhouse.Auth{
+			Username: c.User,
+			Password: c.Password,
+			Database: c.Database,
+		}
+	}
+	if !c.DisableTLS {
+		options.TLS = &tls.Config{InsecureSkipVerify: true} // TODO this should be changed according to `connection.go` (more or less)
+	}
+
 	info := struct {
 		Name    string
 		Version string
 	}{
 		Name:    "quesma",
-		Version: "NEW ODD VERSION", //buildinfo.Version,
+		Version: buildinfo.Version,
 	}
+
 	options.ClientInfo.Products = append(options.ClientInfo.Products, info)
 	return clickhouse.OpenDB(&options), nil
 
 }
 
-func NewClickHouseBackendConnector(endpoint string) *ClickHouseBackendConnector {
+func NewClickHouseBackendConnector(configuration *config.RelationalDbConfiguration) *ClickHouseBackendConnector {
 	return &ClickHouseBackendConnector{
-		Endpoint: endpoint,
+		cfg: configuration,
 	}
 }
 
 // NewClickHouseBackendConnectorWithConnection bridges the gap between the ClickHouseBackendConnector and the sql.DB
 // so that it is can be used in pre-v2 code. Should be removed when moving forwards.
-func NewClickHouseBackendConnectorWithConnection(endpoint string, conn *sql.DB) *ClickHouseBackendConnector {
+func NewClickHouseBackendConnectorWithConnection(_ string, conn *sql.DB) *ClickHouseBackendConnector {
 	return &ClickHouseBackendConnector{
 		BasicSqlBackendConnector: BasicSqlBackendConnector{
 			connection: conn,
 		},
-		Endpoint: endpoint,
 	}
 }
 

quesma/backend_connectors/elasticsearch_backend_connector.go

@@ -28,6 +28,7 @@ type ElasticsearchBackendConnector struct {
 	config config.ElasticsearchConfiguration
 }
 
+// NewElasticsearchBackendConnector is a constructor which uses old (v1) configuration object
 func NewElasticsearchBackendConnector(cfg config.ElasticsearchConfiguration) *ElasticsearchBackendConnector {
 	conn := &ElasticsearchBackendConnector{
 		config: cfg,
@@ -41,6 +42,24 @@ func NewElasticsearchBackendConnector(cfg config.ElasticsearchConfiguration) *El
 	return conn
 }
 
+// NewElasticsearchBackendConnectorFromDbConfig is an alternative constructor which uses the generic database configuration object
+func NewElasticsearchBackendConnectorFromDbConfig(cfg config.RelationalDbConfiguration) *ElasticsearchBackendConnector {
+	conn := &ElasticsearchBackendConnector{
+		config: config.ElasticsearchConfiguration{
+			Url:      cfg.Url,
+			User:     cfg.User,
+			Password: cfg.Password,
+		},
+		client: &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+			Timeout: esRequestTimeout,
+		},
+	}
+	return conn
+}
+
 func (e *ElasticsearchBackendConnector) InstanceName() string {
 	return "elasticsearch" // TODO return name from config
 }
@@ -84,8 +103,12 @@ func (e *ElasticsearchBackendConnector) Send(r *http.Request) (*http.Response, e
 	r.URL.Host = e.config.Url.Host
 	r.URL.Scheme = e.config.Url.Scheme
 	r.RequestURI = "" // this is important for the request to be sent correctly to a different host
-	maybeAuthdReq := elasticsearch.AddBasicAuthIfNeeded(r, e.config.User, e.config.Password)
-	return e.client.Do(maybeAuthdReq)
+	if r.Header.Get("Authorization") == "" {
+		maybeAuthdReq := elasticsearch.AddBasicAuthIfNeeded(r, e.config.User, e.config.Password)
+		return e.client.Do(maybeAuthdReq)
+	} else { // request already came with auth header so just forward these credentials
+		return e.client.Do(r)
+	}
 }
 
 func (e *ElasticsearchBackendConnector) GetId() quesma_api.BackendConnectorType {

quesma/clickhouse/table_discovery.go

@@ -155,7 +155,7 @@ func (td *tableDiscovery) notifyObservers() {
 
 	msg := types.ReloadMessage{Timestamp: time.Now()}
 	for _, observer := range td.reloadObservers {
-		fmt.Println("Sending message to observer", observer)
+		logger.Info().Msgf("Sending message to observer %v", observer)
 		go func() {
 			observer <- msg
 		}()

quesma/frontend_connectors/elasticsearch_ingest.go

@@ -15,9 +15,9 @@ type ElasticsearchIngestFrontendConnector struct {
 	*BasicHTTPFrontendConnector
 }
 
-func NewElasticsearchIngestFrontendConnector(endpoint string, cfg *config.QuesmaConfiguration) *ElasticsearchIngestFrontendConnector {
+func NewElasticsearchIngestFrontendConnector(endpoint string, esCfg config.ElasticsearchConfiguration, disableAuth bool) *ElasticsearchIngestFrontendConnector {
 
-	basicHttpFrontendConnector := NewBasicHTTPFrontendConnector(endpoint, cfg)
+	basicHttpFrontendConnector := NewBasicHTTPFrontendConnector(endpoint, &config.QuesmaConfiguration{Elasticsearch: esCfg, DisableAuth: disableAuth})
 	basicHttpFrontendConnector.responseMutator = func(w http.ResponseWriter) http.ResponseWriter {
 		w.Header().Set("Content-Type", "application/json")
 		return w

quesma/frontend_connectors/elasticsearch_query.go

@@ -15,9 +15,9 @@ type ElasticsearchQueryFrontendConnector struct {
 	*BasicHTTPFrontendConnector
 }
 
-func NewElasticsearchQueryFrontendConnector(endpoint string, cfg *config.QuesmaConfiguration) *ElasticsearchQueryFrontendConnector {
-
-	basicHttpFrontendConnector := NewBasicHTTPFrontendConnector(endpoint, cfg)
+func NewElasticsearchQueryFrontendConnector(endpoint string, esCfg config.ElasticsearchConfiguration, disableAuth bool) *ElasticsearchQueryFrontendConnector {
+	// The esCfg is here only for F/E auth purposes and should probably not be wrapped againa in `config.QuesmaConfiguration`
+	basicHttpFrontendConnector := NewBasicHTTPFrontendConnector(endpoint, &config.QuesmaConfiguration{Elasticsearch: esCfg, DisableAuth: disableAuth})
 	basicHttpFrontendConnector.responseMutator = func(w http.ResponseWriter) http.ResponseWriter {
 		w.Header().Set("Content-Type", "application/json")
 		return w

quesma/processors/es_to_ch_common/common.go

@@ -8,11 +8,13 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/clickhouse"
 	"github.com/QuesmaOrg/quesma/quesma/common_table"
 	"github.com/QuesmaOrg/quesma/quesma/ingest"
+	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/persistence"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/ui"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
 	"github.com/QuesmaOrg/quesma/quesma/table_resolver"
+	"github.com/QuesmaOrg/quesma/quesma/telemetry"
 	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"github.com/ucarion/urlpath"
 	"net/http"
@@ -98,9 +100,11 @@ type LegacyQuesmaDependencies struct {
 	TableDiscovery      clickhouse.TableDiscovery
 	SchemaRegistry      schema.Registry
 	TableResolver       table_resolver.TableResolver
-	Adminconsole        *ui.QuesmaManagementConsole
+	UIConsole           *ui.QuesmaManagementConsole
 	AbTestingController *sender.SenderCoordinator
 	IngestProcessor     *ingest.IngestProcessor
+	LogManager          clickhouse.LogManagerIFace
+	LogChan             <-chan logger.LogWithLevel
 }
 
 func newLegacyQuesmaDependencies(
@@ -113,6 +117,9 @@ func newLegacyQuesmaDependencies(
 	tableResolver table_resolver.TableResolver,
 	abTestingController *sender.SenderCoordinator,
 	ingestProcessor *ingest.IngestProcessor,
+	logManager clickhouse.LogManagerIFace,
+	logChan <-chan logger.LogWithLevel,
+	uiConsole *ui.QuesmaManagementConsole,
 ) *LegacyQuesmaDependencies {
 	return &LegacyQuesmaDependencies{
 		DependenciesImpl:    baseDependencies,
@@ -124,21 +131,27 @@ func newLegacyQuesmaDependencies(
 		TableResolver:       tableResolver,
 		AbTestingController: abTestingController,
 		IngestProcessor:     ingestProcessor,
+		LogManager:          logManager,
+		LogChan:             logChan,
+		UIConsole:           uiConsole,
 	}
 }
 
-func InitializeLegacyQuesmaDependencies(baseDeps *quesma_api.DependenciesImpl, oldQuesmaConfig *config.QuesmaConfiguration) *LegacyQuesmaDependencies {
+func InitializeLegacyQuesmaDependencies(baseDeps *quesma_api.DependenciesImpl, oldQuesmaConfig *config.QuesmaConfiguration, logChan <-chan logger.LogWithLevel) *LegacyQuesmaDependencies {
 	connectionPool := clickhouse.InitDBConnectionPool(oldQuesmaConfig)
 	virtualTableStorage := persistence.NewElasticJSONDatabase(oldQuesmaConfig.Elasticsearch, common_table.VirtualTableElasticIndexName)
 	tableDisco := clickhouse.NewTableDiscovery(oldQuesmaConfig, connectionPool, virtualTableStorage)
 	schemaRegistry := schema.NewSchemaRegistry(clickhouse.TableDiscoveryTableProviderAdapter{TableDiscovery: tableDisco}, oldQuesmaConfig, clickhouse.SchemaTypeAdapter{})
 	schemaRegistry.Start()
 	dummyTableResolver := table_resolver.NewDummyTableResolver(oldQuesmaConfig.IndexConfig, oldQuesmaConfig.UseCommonTableForWildcard)
+	//phoneHomeAgent := baseDeps.PhoneHomeAgent() //TODO perhaps remove? we could get away with Client if not the UI console. Because of that we have to use Agent
+	phoneHomeAgent := telemetry.NewPhoneHomeAgent(oldQuesmaConfig, connectionPool, "DuMMY_CLIENT_ID")
+	phoneHomeAgent.Start()
 
 	ingestProcessor := ingest.NewIngestProcessor(
 		oldQuesmaConfig,
 		connectionPool,
-		baseDeps.PhoneHomeAgent(),
+		phoneHomeAgent,
 		tableDisco,
 		schemaRegistry,
 		virtualTableStorage,
@@ -149,6 +162,12 @@ func InitializeLegacyQuesmaDependencies(baseDeps *quesma_api.DependenciesImpl, o
 	abTestingController := sender.NewSenderCoordinator(oldQuesmaConfig, ingestProcessor)
 	abTestingController.Start()
 
-	legacyDependencies := newLegacyQuesmaDependencies(*baseDeps, oldQuesmaConfig, connectionPool, *virtualTableStorage, tableDisco, schemaRegistry, dummyTableResolver, abTestingController, ingestProcessor)
+	logManager := clickhouse.NewEmptyLogManager(oldQuesmaConfig, connectionPool, phoneHomeAgent, tableDisco)
+	logManager.Start()
+
+	quesmaManagementConsole := ui.NewQuesmaManagementConsole(oldQuesmaConfig, logManager, logChan, phoneHomeAgent, schemaRegistry, dummyTableResolver)
+	go quesmaManagementConsole.Run()
+
+	legacyDependencies := newLegacyQuesmaDependencies(*baseDeps, oldQuesmaConfig, connectionPool, *virtualTableStorage, tableDisco, schemaRegistry, dummyTableResolver, abTestingController, ingestProcessor, logManager, logChan, quesmaManagementConsole)
 	return legacyDependencies
 }

quesma/processors/es_to_ch_query/elasticsearch_to_clickhouse_query_processor.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 	"github.com/QuesmaOrg/quesma/quesma/backend_connectors"
-	"github.com/QuesmaOrg/quesma/quesma/clickhouse"
 	"github.com/QuesmaOrg/quesma/quesma/elasticsearch"
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/processors"
@@ -70,12 +69,10 @@ func (p *ElasticsearchToClickHouseQueryProcessor) GetId() string {
 // which uses `quesma_api.BackendConnector` instead of `*sql.DB` for the database connection.
 func (p *ElasticsearchToClickHouseQueryProcessor) prepareTemporaryQueryProcessor() *quesm.QueryRunner {
 
-	logManager := clickhouse.NewEmptyLogManager(p.legacyDependencies.OldQuesmaConfig, p.legacyDependencies.ConnectionPool, p.legacyDependencies.PhoneHomeAgent(), p.legacyDependencies.TableDiscovery)
-	logManager.Start()
-
-	queryRunner := quesm.NewQueryRunner(logManager,
+	queryRunner := quesm.NewQueryRunner(
+		p.legacyDependencies.LogManager,
 		p.legacyDependencies.OldQuesmaConfig,
-		nil,
+		p.legacyDependencies.UIConsole,
 		p.legacyDependencies.SchemaRegistry,
 		p.legacyDependencies.AbTestingController.GetSender(),
 		p.legacyDependencies.TableResolver,
@@ -270,20 +267,9 @@ func (p *ElasticsearchToClickHouseQueryProcessor) GetSupportedBackendConnectors(
 }
 
 func findQueryTarget(index string, processorConfig config.QuesmaProcessorConfig) string {
-	var defaultTargetFromConfig string
-	wildcardConfig, ok := processorConfig.IndexConfig["*"]
-	if !ok {
-		logger.Warn().Msgf("No wildcard index config found in processor config!!")
-		return config.ClickhouseTarget
-	}
-	if len(wildcardConfig.QueryTarget) == 0 {
-		logger.Warn().Msgf("wildcard index has no target!!")
-		return config.ClickhouseTarget
-	}
-	defaultTargetFromConfig = wildcardConfig.QueryTarget[0]
 	_, found := processorConfig.IndexConfig[index]
 	if !found {
-		return defaultTargetFromConfig
+		return processorConfig.DefaultTargetConnectorType
 	} else { // per legacy syntax, if present means it's a clickhouse target
 		return config.ClickhouseTarget
 	}

quesma/quesma/config/config.go

@@ -24,6 +24,10 @@ var (
 	telemetryUrl = &Url{Scheme: "https", Host: "api.quesma.com", Path: "/phone-home"}
 )
 
+func DefaultTelemetryUrl() *Url {
+	return telemetryUrl
+}
+
 type QuesmaConfiguration struct {
 	// both clickhouse and hydrolix connections are going to be deprecated and everything is going to live under connector
 	Connectors       map[string]RelationalDbConfiguration