Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Fix Lucene query behavior (match operator) #1426

Merged
merged 9 commits into from
May 13, 2025
Merged

Fix Lucene query behavior (match operator) #1426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
144df28
Fix lucene syntax
mieciu May 12, 2025
71011c7
unit test adjustments
mieciu May 12, 2025
2be52b5
unit test adjustments v2
mieciu May 12, 2025
e926fab
lucene fix cont'd
mieciu May 12, 2025
a6a5c3f
roll comment to next line
mieciu May 12, 2025
a291230
comment
mieciu May 13, 2025
380331e
other tests
mieciu May 13, 2025
a11cc61
tests cont'd
mieciu May 13, 2025
5fbb9cc
Merge branch 'main' into fix-lucene-syntax
mieciu May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions platform/frontend_connectors/schema_transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1136,10 +1136,12 @@ func (s *SchemaCheckPass) applyMatchOperator(indexSchema schema.Schema, query *m
case schema.QuesmaTypeKeyword.Name:
return equal()
default:
// ILIKE '%%' has terrible performance, but semantically means "is not null", hence this transformation
if rhsValue == "%%" {
if rhsValue == "%%" { // ILIKE '%%' has terrible performance, but semantically means "is not null", hence this transformation
return model.NewInfixExpr(lhs, "IS", model.NewLiteral("NOT NULL"))
}
// we might investigate the potential performance gain of checking
// that if rhsValue doesn't contain '%' we could use '=' instead of 'ILIKE'
// *however* that'd require few tweaks in the parser
return ilike()
}
}
Expand Down
94 changes: 47 additions & 47 deletions platform/parsers/elastic_query_dsl/lucene/lucene_parser_test.go
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 0 additions & 3 deletions platform/parsers/elastic_query_dsl/lucene/value.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ func (v termValue) toExpression(fieldName string) model.Expr {
if alreadyQuoted(v.term) {
termAsStringToClickhouse = termAsStringToClickhouse[1 : len(termAsStringToClickhouse)-1]
}
if !util.IsSurroundedWithPercents(termAsStringToClickhouse) {
termAsStringToClickhouse = util.SurroundWithPercents(termAsStringToClickhouse)
}
if !util.IsSingleQuoted(termAsStringToClickhouse) {
termAsStringToClickhouse = util.SingleQuote(termAsStringToClickhouse)
}
Expand Down
4 changes: 2 additions & 2 deletions platform/testdata/clients/clover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ var CloverTests = []testdata.AggregationTestCase{
ExpectedPancakeSQL: `
SELECT count(*) AS "aggr__timeseries__count",
countIf(true) AS "metric__timeseries__a2-denominator_col_0",
countIf(NOT ("table.flower" __quesma_match '%clover%')) AS
countIf(NOT ("table.flower" __quesma_match 'clover')) AS
"metric__timeseries__a2-numerator_col_0"
FROM __quesma_table_name
WHERE ("@timestamp">=fromUnixTimestamp64Milli(1728640683723) AND "@timestamp"<=
Expand Down Expand Up @@ -1115,7 +1115,7 @@ var CloverTests = []testdata.AggregationTestCase{
"aggr__q__time_buckets__key_0", count(*) AS "aggr__q__time_buckets__count",
sumOrNull("count") AS "metric__q__time_buckets__sum(count)_col_0"
FROM __quesma_table_name
WHERE NOT ("str_field" __quesma_match '%CRASH%')
WHERE NOT ("str_field" __quesma_match 'CRASH')
GROUP BY toInt64((toUnixTimestamp64Milli("@timestamp")+timeZoneOffset(toTimezone
("@timestamp", 'Europe/Warsaw'))*1000) / 1800000) AS
"aggr__q__time_buckets__key_0"
Expand Down
8 changes: 4 additions & 4 deletions platform/testdata/kibana_sample_data_ecommerce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -858,11 +858,11 @@ var KibanaSampleDataEcommerce = []AggregationTestCase{
SELECT toInt64((toUnixTimestamp64Milli("order_date")+timeZoneOffset(toTimezone(
"order_date", 'Europe/Warsaw'))*1000) / 43200000) AS "aggr__0__key_0",
count(*) AS "aggr__0__count",
countIf(("products.product_name" __quesma_match '%%cocktail%' OR
"__quesma_fulltext_field_name" __quesma_match '%dress%%')) AS
countIf(("products.product_name" __quesma_match '%cocktail' OR
"__quesma_fulltext_field_name" __quesma_match 'dress%')) AS
"aggr__0__1-bucket__count",
sumOrNullIf("taxful_total_price", ("products.product_name" __quesma_match '%%cocktail%'
OR "__quesma_fulltext_field_name" __quesma_match '%dress%%')) AS
sumOrNullIf("taxful_total_price", ("products.product_name" __quesma_match '%cocktail'
OR "__quesma_fulltext_field_name" __quesma_match 'dress%')) AS
"metric__0__1-bucket__1-metric_col_0"
FROM __quesma_table_name
WHERE ("order_date">=fromUnixTimestamp64Milli(1740234098238) AND "order_date"<=
Expand Down
38 changes: 19 additions & 19 deletions platform/testdata/kibana_sample_data_flights.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,8 +800,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%' AND
"Cancelled" __quesma_match '%true%'))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true' AND
"Cancelled" __quesma_match 'true'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS "aggr__1__2__key_0"
ORDER BY "aggr__1__2__key_0" ASC`,
Expand All @@ -813,8 +813,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%'))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -832,8 +832,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%')))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__4_col_0", "top_metrics__1__2__4_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand All @@ -847,8 +847,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%'))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -866,8 +866,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%')))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__5_col_0", "top_metrics__1__2__5_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand All @@ -881,8 +881,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%'))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -900,8 +900,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%')))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__6_col_0", "top_metrics__1__2__6_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand All @@ -915,8 +915,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%'))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -934,8 +934,8 @@ var KibanaSampleDataFlights = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match '%true%'
AND "Cancelled" __quesma_match '%true%')))
fromUnixTimestamp64Milli(1740835408853)) AND ("FlightDelay" __quesma_match 'true'
AND "Cancelled" __quesma_match 'true')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__7_col_0", "top_metrics__1__2__7_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand Down Expand Up @@ -1638,7 +1638,7 @@ var KibanaSampleDataFlights = []AggregationTestCase{
SELECT toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS "aggr__0__key_0",
count(*) AS "aggr__0__count",
countIf("FlightDelay" __quesma_match '%true%') AS "metric__0__1-bucket_col_0",
countIf("FlightDelay" __quesma_match 'true') AS "metric__0__1-bucket_col_0",
countIf("__quesma_fulltext_field_name" __quesma_match '%') AS "metric__0__2-bucket_col_0"
FROM __quesma_table_name
WHERE ("timestamp">=fromUnixTimestamp64Milli(1740230608853) AND "timestamp"<=
Expand Down
20 changes: 10 additions & 10 deletions platform/testdata/kibana_sample_data_logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,8 @@ var KibanaSampleDataLogs = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740178800000) AND "timestamp"<=
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match '%error%' AND
"tags" __quesma_match '%security%'))
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match 'error' AND
"tags" __quesma_match 'security'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS "aggr__1__2__key_0"
ORDER BY "aggr__1__2__key_0" ASC`,
Expand All @@ -362,8 +362,8 @@ var KibanaSampleDataLogs = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740178800000) AND "timestamp"<=
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match '%error%' AND
"tags" __quesma_match '%security%'))
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match 'error' AND
"tags" __quesma_match 'security'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -381,8 +381,8 @@ var KibanaSampleDataLogs = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740178800000) AND "timestamp"<=
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match '%error%' AND
"tags" __quesma_match '%security%')))
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match 'error' AND
"tags" __quesma_match 'security')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__4_col_0", "top_metrics__1__2__4_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand All @@ -396,8 +396,8 @@ var KibanaSampleDataLogs = []AggregationTestCase{
count(*) AS "aggr__1__2__count"
FROM __quesma_table_name
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740178800000) AND "timestamp"<=
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match '%error%' AND
"tags" __quesma_match '%security%'))
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match 'error' AND
"tags" __quesma_match 'security'))
GROUP BY toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(
toTimezone("timestamp", 'Europe/Warsaw'))*1000) / 10800000) AS
"aggr__1__2__key_0"
Expand All @@ -415,8 +415,8 @@ var KibanaSampleDataLogs = []AggregationTestCase{
toInt64((toUnixTimestamp64Milli("timestamp")+timeZoneOffset(toTimezone(
"timestamp", 'Europe/Warsaw'))*1000) / 10800000))
WHERE (("timestamp">=fromUnixTimestamp64Milli(1740178800000) AND "timestamp"<=
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match '%error%' AND
"tags" __quesma_match '%security%')))
fromUnixTimestamp64Milli(1740831278103)) AND ("tags" __quesma_match 'error' AND
"tags" __quesma_match 'security')))
SELECT "aggr__1__count", "aggr__1__2__key_0", "aggr__1__2__count",
"top_metrics__1__2__5_col_0", "top_metrics__1__2__5_col_1", "top_hits_rank"
FROM "quesma_top_hits_join"
Expand Down
12 changes: 6 additions & 6 deletions platform/testdata/requests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1252,9 +1252,9 @@ var TestsSearch = []SearchTestCase{
},
"track_total_hits": false
}`,
[]string{`"exception-list-agnostic.list_id" __quesma_match '%endpoint\_event\_filters%'`},
[]string{`"exception-list-agnostic.list_id" __quesma_match 'endpoint\_event\_filters'`},
model.ListAllFields,
[]string{`SELECT "message" FROM ` + TableName + ` WHERE "exception-list-agnostic.list_id"='%endpoint\\_event\\_filters%'`},
[]string{`SELECT "message" FROM ` + TableName + ` WHERE "exception-list-agnostic.list_id"='endpoint\\_event\\_filters'`},
[]string{},
},
{ // [10]
Expand All @@ -1279,9 +1279,9 @@ var TestsSearch = []SearchTestCase{
},
"track_total_hits": false
}`,
[]string{fullTextFieldName + ` __quesma_match '%ingest-agent-policies%'`},
[]string{fullTextFieldName + ` __quesma_match 'ingest-agent-policies'`},
model.ListAllFields,
[]string{`SELECT "message" FROM ` + TableName + ` WHERE ` + fullTextFieldName + ` ILIKE '%ingest-agent-policies%'`},
[]string{`SELECT "message" FROM ` + TableName + ` WHERE ` + fullTextFieldName + ` ILIKE 'ingest-agent-policies'`},
[]string{},
},
{ // [11]
Expand Down Expand Up @@ -1377,9 +1377,9 @@ var TestsSearch = []SearchTestCase{
"track_total_hits": false,
"size": 1
}`,
[]string{`"message" __quesma_match '%% logged%'`},
[]string{`"message" __quesma_match '% logged'`},
model.ListAllFields,
[]string{`SELECT "message" FROM ` + TableName + ` WHERE "message" ILIKE '%% logged%'`},
[]string{`SELECT "message" FROM ` + TableName + ` WHERE "message" ILIKE '% logged'`},
[]string{},
},
{ // [16]
Expand Down
Loading