WIP

Author: nablaone

platform/clickhouse/type_adapter.go

@@ -22,8 +22,8 @@ func (c SchemaTypeAdapter) Convert(s string) (schema.QuesmaType, bool) {
 	}
 
 	switch s {
-	case "String":
-		return schema.QuesmaTypeText, true
+	case "String": // This should be treated as a text type (full text search). But ingest do not distinguish between LowCardinality and String.
+		return schema.QuesmaTypeKeyword, true
 	case "LowCardinality(String)", "UUID", "FixedString":
 		return schema.QuesmaTypeKeyword, true
 	case "Int", "Int8", "Int16", "Int32", "Int64":

platform/schema/registry.go

@@ -326,6 +326,12 @@ func (s *schemaRegistry) populateSchemaFromTableDefinition(definitions map[strin
 			}
 			if existing, exists := fields[propertyName]; !exists {
 				if quesmaType, resolved := s.dataSourceTypeAdapter.Convert(column.Type); resolved {
+
+					// in most cases `message` field is a text field, it can be used for full-text search
+					if quesmaType.Name == QuesmaTypeKeyword.Name && column.Name == "message" {
+						quesmaType = QuesmaTypeText // special case for message field, we want it to be text
+					}
+
 					fields[propertyName] = Field{PropertyName: propertyName, InternalPropertyName: FieldName(column.Name), InternalPropertyType: column.Type, Type: quesmaType, Origin: column.Origin}
 				} else {
 					logger.Debug().Msgf("type %s not supported, falling back to keyword", column.Type)