Merge branch 'main' into datetime

Author: trzysiek

docs/public/docs/limitations.md

@@ -38,7 +38,7 @@ Currently supported:
   including: `boolean`, `match`, `match phrase`, `multi-match`, `query string`, `nested`, `match all`, `exists`, `prefix`, `range`, `term`, `terms`, `wildcard`
 - most popular [Aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html),
   including: `avg`, `cardinality`, `max`, `min`, `percentile ranks`, `percentiles`, `stats`, `sum`, `top hits`, `top metrics`, `value count`,
-  `date histogram`, `date range`, `filter`, `filters`, `histogram`, `range`, `singificant terms`, `terms`, `ip prefix`, `ip range`, `geo_bounds`, `geohash_grid`
+  `date histogram`, `date range`, `filter`, `filters`, `histogram`, `range`, `significant terms`, `terms`, `ip prefix`, `ip range`, `geo_bounds`, `geohash_grid`
 
 Which as a result allows you to run Kibana/OSD queries and dashboards on data residing in ClickHouse/Hydrolix.
 

examples/kibana-sample-data/docker-compose.yml

@@ -76,7 +76,7 @@ services:
     command: [ "/bin/bash", "-c", "/local_mount/run.sh" ]
   clickhouse:
     # user: 'default', no password
-    image: clickhouse/clickhouse-server:23.12.2.59-alpine
+    image: clickhouse/clickhouse-server:24.5.3.5-alpine
     ports:
       - "8123:8123"
       - "9000:9000"

examples/kibana-sample-data/kibana/run.sh

@@ -61,7 +61,7 @@ add_sample_dataset() {
 wait_until_available
 
 add_sample_dataset "flights"
-#add_sample_dataset "logs"
+add_sample_dataset "logs"
 add_sample_dataset "ecommerce"
 
 echo -n "Adding data view logs-generic... "

examples/kibana-sample-data/quesma/config/local-dev.yaml

@@ -54,6 +54,21 @@ processors:
                 type: geo_point
               "OriginLocation":
                 type: geo_point
+        kibana_sample_data_logs:
+          target: [ my-clickhouse-data-source ]
+          schemaOverrides:
+            fields:
+              timestamp:
+                type: alias
+                targetColumnName: "@timestamp"
+              ip:
+                type: ip
+              clientip:
+                type: ip
+              "geo.coordinates":
+                type: geo_point
+              "geo.src":
+                type: keyword
         logs-generic-default:
           target:
             - my-clickhouse-data-source
@@ -103,6 +118,21 @@ processors:
                 type: geo_point
               "OriginLocation":
                 type: geo_point
+        kibana_sample_data_logs:
+          target: [ my-clickhouse-data-source ]
+          schemaOverrides:
+            fields:
+              timestamp:
+                type: alias
+                targetColumnName: "@timestamp"
+              ip:
+                type: ip
+              clientip:
+                type: ip
+              "geo.coordinates":
+                type: geo_point
+              "geo.src":
+                type: keyword
         logs-generic-default:
           target:
             - my-clickhouse-data-source

platform/config/config_v2.go

@@ -115,14 +115,22 @@ type Processor struct {
 const DefaultWildcardIndexName = "*"
 
 // Configuration of QuesmaV1ProcessorQuery and QuesmaV1ProcessorIngest
-type QuesmaProcessorConfig struct {
-	UseCommonTable bool           `koanf:"useCommonTable"`
-	IndexConfig    IndicesConfigs `koanf:"indexes"`
-	// DefaultTargetConnectorType is used in V2 code only
-	DefaultTargetConnectorType string //it is not serialized to maintain configuration BWC, so it's basically just populated from '*' config in `config_v2.go`
-}
+type (
+	QuesmaProcessorConfig struct {
+		UseCommonTable bool           `koanf:"useCommonTable"`
+		IndexConfig    IndicesConfigs `koanf:"indexes"`
+		// DefaultTargetConnectorType is used in V2 code only
+		DefaultTargetConnectorType string //it is not serialized to maintain configuration BWC, so it's basically just populated from '*' config in `config_v2.go`
+	}
+	IndicesConfigs map[string]IndexConfiguration
+)
 
-type IndicesConfigs map[string]IndexConfiguration
+func (p *QuesmaProcessorConfig) IsFieldMapSyntaxEnabled(indexName string) bool {
+	if indexConf, exists := p.IndexConfig[indexName]; exists {
+		return indexConf.EnableFieldMapSyntax
+	}
+	return false
+}
 
 func LoadV2Config() QuesmaNewConfiguration {
 	var v2config QuesmaNewConfiguration

platform/frontend_connectors/route_handlers.go

@@ -162,8 +162,9 @@ func HandleGetIndex(sr schema.Registry, index string) (*quesma_api.Result, error
 	return getIndexResult(index, mappings)
 }
 
-func HandleTermsEnum(ctx context.Context, indexPattern string, body types.JSON, lm clickhouse.LogManagerIFace, sr schema.Registry, dependencies quesma_api.Dependencies) (*quesma_api.Result, error) {
-	if responseBody, err := terms_enum.HandleTermsEnum(ctx, indexPattern, body, lm, sr, dependencies.DebugInfoCollector()); err != nil {
+func HandleTermsEnum(ctx context.Context, indexPattern string, body types.JSON, lm clickhouse.LogManagerIFace, sr schema.Registry,
+	isFieldMapSyntaxEnabled bool, dependencies quesma_api.Dependencies) (*quesma_api.Result, error) {
+	if responseBody, err := terms_enum.HandleTermsEnum(ctx, indexPattern, body, lm, sr, isFieldMapSyntaxEnabled, dependencies.DebugInfoCollector()); err != nil {
 		return nil, err
 	} else {
 		return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil

platform/frontend_connectors/router_test.go

@@ -391,7 +391,8 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
 				return nil, errors.New("invalid request body, expecting JSON")
 			}
 
-			if responseBody, err := terms_enum.HandleTermsEnum(ctx, req.Params["index"], body, lm, sr, console); err != nil {
+			const isFieldMapSyntaxEnabled = false
+			if responseBody, err := terms_enum.HandleTermsEnum(ctx, req.Params["index"], body, lm, sr, isFieldMapSyntaxEnabled, console); err != nil {
 				return nil, err
 			} else {
 				return elasticsearchQueryResult(string(responseBody), http.StatusOK), nil

platform/frontend_connectors/router_v2.go

@@ -251,7 +251,12 @@ func ConfigureSearchRouterV2(cfg *config.QuesmaConfiguration, dependencies quesm
 		if err != nil {
 			return nil, errors.New("invalid request body, expecting JSON")
 		}
-		return HandleTermsEnum(ctx, indexPattern, body, lm, sr, dependencies)
+
+		var isFieldMapSyntaxEnabled bool
+		if indexCfg, exists := cfg.IndexConfig[indexPattern]; exists {
+			isFieldMapSyntaxEnabled = indexCfg.EnableFieldMapSyntax
+		}
+		return HandleTermsEnum(ctx, indexPattern, body, lm, sr, isFieldMapSyntaxEnabled, dependencies)
 	})
 
 	router.Register(routes.EQLSearch, and(method("GET", "POST"), matchedAgainstPattern(tableResolver)), func(ctx context.Context, req *quesma_api.Request, _ http.ResponseWriter) (*quesma_api.Result, error) {

platform/frontend_connectors/schema_map_transformation.go

@@ -50,8 +50,8 @@ func (v *mapTypeResolver) isMap(fieldName string) (exists bool, scope searchScop
 	if !ok {
 		return false, scope, fieldName
 	}
-	col, ok := v.indexSchema.Fields[schema.FieldName(tableColumnName.InternalPropertyName.AsString())]
 
+	col, ok := v.indexSchema.Fields[schema.FieldName(tableColumnName.InternalPropertyName.AsString())]
 	if ok {
 		if strings.HasPrefix(col.InternalPropertyType, "Map") {
 			return true, scope, tableColumnName.InternalPropertyName.AsString()
@@ -72,12 +72,10 @@ func NewMapTypeVisitor(resolver mapTypeResolver) model.ExprVisitor {
 	visitor := model.NewBaseVisitor()
 
 	visitor.OverrideVisitColumnRef = func(b *model.BaseExprVisitor, e model.ColumnRef) interface{} {
-		isMap, _, realName := resolver.isMap(e.ColumnName)
-		if !isMap {
-			return e
+		if isMap, _, realName := resolver.isMap(e.ColumnName); isMap {
+			return model.NewColumnRefWithTable(realName, e.TableAlias)
 		}
-
-		return model.NewColumnRef(realName)
+		return e.Clone()
 	}
 
 	visitor.OverrideVisitInfix = func(b *model.BaseExprVisitor, e model.InfixExpr) interface{} {

platform/frontend_connectors/schema_transformer.go

@@ -15,6 +15,7 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/util"
 	"github.com/k0kubun/pp"
 	"slices"
+	"github.com/QuesmaOrg/quesma/platform/transformations"
 	"sort"
 	"strings"
 )
@@ -714,6 +715,7 @@ func (s *SchemaCheckPass) applyTimestampField(ctx context.Context, indexSchema s
 
 }
 
+<<<<<<< HEAD
 func (s *SchemaCheckPass) applyTimestampFieldd(ctx context.Context, indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
 	table, ok := s.tableDiscovery.TableDefinitions().Load(query.TableName)
 	if !ok {
@@ -1015,6 +1017,7 @@ func (s *SchemaCheckPass) applyFieldMapSyntax(ctx context.Context, indexSchema s
 }
 
 func (s *SchemaCheckPass) applyFieldEncoding(ctx context.Context, indexSchema schema.Schema, query *model.Query) (*model.Query, error) {
+
 	table, ok := s.tableDiscovery.TableDefinitions().Load(query.TableName)
 	if !ok {
 		return nil, fmt.Errorf("table %s not found", query.TableName)
@@ -1280,7 +1283,9 @@ func (s *SchemaCheckPass) Transform(ctx context.Context, queries []*model.Query)
 		{TransformationName: "PhysicalFromExpressionTransformation", Transformation: s.applyPhysicalFromExpression},
 		{TransformationName: "WildcardExpansion", Transformation: s.applyWildcardExpansion},
 		{TransformationName: "RuntimeMappings", Transformation: s.applyRuntimeMappings},
-		{TransformationName: "FieldMapSyntaxTransformation", Transformation: s.applyFieldMapSyntax},
+		{TransformationName: "AllNecessaryCommonTransformations", Transformation: func(schema schema.Schema, query *model.Query) (*model.Query, error) {
+			return transformations.ApplyAllNecessaryCommonTransformations(query, schema, s.cfg.MapFieldsDiscoveringEnabled)
+		}},
 		{TransformationName: "AliasColumnsTransformation", Transformation: s.applyAliasColumns},
 		{TransformationName: "UnixTimestampToDateTimeTransformation", Transformation: s.applyTimestampFieldd},
 
@@ -1411,7 +1416,6 @@ func (s *SchemaCheckPass) applyMatchOperator(ctx context.Context, indexSchema sc
 				return model.NewInfixExpr(lhs, "ILIKE", rhs.Clone())
 			}
 			equal := func() model.Expr {
-				rhsValue = strings.Trim(rhsValue, "%")
 				return model.NewInfixExpr(lhs, "=", rhs.Clone())
 			}
 
@@ -1428,6 +1432,8 @@ func (s *SchemaCheckPass) applyMatchOperator(ctx context.Context, indexSchema sc
 			// TODO: improve? we seem to be `ilike'ing` too much
 			switch field.Type.String() {
 			case schema.QuesmaTypeInteger.Name, schema.QuesmaTypeLong.Name, schema.QuesmaTypeUnsignedLong.Name, schema.QuesmaTypeFloat.Name, schema.QuesmaTypeBoolean.Name:
+				rhs.Value = strings.Trim(rhsValue, "%")
+				rhs.EscapeType = model.NormalNotEscaped
 				return equal()
 			default:
 				return ilike()