Omit non-existing indexes (#1173)

When we query multiple indexes, we may treat non-existent indexes as
empty ones. This solves the case when we store daily based indexes in
the common table and query with Grafana.

Author: nablaone

quesma/quesma/search.go

@@ -243,7 +243,6 @@ func (q *QueryRunner) HandleMultiSearch(ctx context.Context, defaultIndexName st
 					StatusCode:    http.StatusInternalServerError,
 					GenericResult: queryparser.BadRequestParseError(err),
 				}
-				return nil, err
 			}
 
 			responses = append(responses, wrappedErr)
@@ -505,7 +504,7 @@ func (q *QueryRunner) handleSearchCommon(ctx context.Context, indexPattern strin
 		for _, indexName := range resolvedIndexes {
 			table, _ = tables.Load(q.cfg.IndexConfig[indexName].TableName(indexName))
 			if table == nil {
-				return []byte{}, end_user_errors.ErrNoSuchTable.New(fmt.Errorf("can't load %s table", indexName)).Details("Table: %s", indexName)
+				continue
 			}
 			if table.VirtualTable {
 				virtualOnlyTables = append(virtualOnlyTables, indexName)

quesma/table_resolver/rules.go

@@ -134,8 +134,14 @@ func makeDefaultWildcard(quesmaConf config.QuesmaConfiguration, pipeline string)
 		for _, target := range targets {
 			switch target {
 			case config.ClickhouseTarget:
+				var tableName string
+				if quesmaConf.UseCommonTableForWildcard {
+					tableName = common_table.TableName
+				} else {
+					tableName = resolveTableName(quesmaConf, part)
+				}
 				useConnectors = append(useConnectors, &quesma_api.ConnectorDecisionClickhouse{
-					ClickhouseTableName: resolveTableName(quesmaConf, part),
+					ClickhouseTableName: tableName,
 					IsCommonTable:       quesmaConf.UseCommonTableForWildcard,
 					ClickhouseIndexes:   []string{part},
 				})

quesma/table_resolver/table_resolver_test.go

@@ -57,6 +57,8 @@ func TestTableResolver(t *testing.T) {
 
 	cfg := config.QuesmaConfiguration{IndexConfig: indexConf, DefaultQueryTarget: []string{config.ElasticsearchTarget}, DefaultIngestTarget: []string{config.ElasticsearchTarget}}
 
+	cfgClickhouseOnlyUseCommonTable := config.QuesmaConfiguration{IndexConfig: indexConf, DefaultQueryTarget: []string{config.ClickhouseTarget}, DefaultIngestTarget: []string{config.ClickhouseTarget}, UseCommonTableForWildcard: true}
+
 	tests := []struct {
 		name              string
 		pipeline          string
@@ -66,6 +68,7 @@ func TestTableResolver(t *testing.T) {
 		virtualTables     []string
 		indexConf         map[string]config.IndexConfiguration
 		expected          mux.Decision
+		quesmaConf        *config.QuesmaConfiguration
 	}{
 		{
 			name:     "elastic fallback",
@@ -266,6 +269,21 @@ func TestTableResolver(t *testing.T) {
 			},
 			indexConf: indexConf,
 		},
+		{
+			name:          "query pattern (not existing virtual table)",
+			pipeline:      mux.QueryPipeline,
+			pattern:       "common-index1,common-index2",
+			virtualTables: []string{"common-index1"},
+			expected: mux.Decision{
+				UseConnectors: []mux.ConnectorDecision{&mux.ConnectorDecisionClickhouse{
+					ClickhouseTableName: common_table.TableName,
+					ClickhouseIndexes:   []string{"common-index1", "common-index2"},
+					IsCommonTable:       true,
+				}},
+			},
+			indexConf:  indexConf,
+			quesmaConf: &cfgClickhouseOnlyUseCommonTable,
+		},
 		{
 			name:     "query kibana internals",
 			pipeline: mux.QueryPipeline,
@@ -377,6 +395,12 @@ func TestTableResolver(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+
+			currentQuesmaConf := cfg
+			if tt.quesmaConf != nil {
+				currentQuesmaConf = *tt.quesmaConf
+			}
+
 			tableDiscovery := clickhouse.NewEmptyTableDiscovery()
 
 			for _, index := range tt.clickhouseIndexes {
@@ -394,7 +418,7 @@ func TestTableResolver(t *testing.T) {
 
 			elasticResolver := elasticsearch.NewFixedIndexManagement(tt.elasticIndexes...)
 
-			resolver := NewTableResolver(cfg, tableDiscovery, elasticResolver)
+			resolver := NewTableResolver(currentQuesmaConf, tableDiscovery, elasticResolver)
 
 			decision := resolver.Resolve(tt.pipeline, tt.pattern)