Skip to content

Commit 1eb4f58

Browse files
johansolbakkenjohansolbakken
authored
Merge pull request #34523 from vespa-engine/johsol/revert-revert-grouping-filter-predicates
Add support for `and`, `or` and `not` predicates to filter in grouping
2 parents b2dadda + ff641d1 commit 1eb4f58

33 files changed

Lines changed: 1058 additions & 42 deletions

File tree

‎container-search/abi-spec.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2636,6 +2636,20 @@
26362636
],
26372637
"fields" : [ ]
26382638
},
2639+
"com.yahoo.search.grouping.request.AndPredicate" : {
2640+
"superClass" : "com.yahoo.search.grouping.request.FilterExpression",
2641+
"interfaces" : [ ],
2642+
"attributes" : [
2643+
"public"
2644+
],
2645+
"methods" : [
2646+
"public void <init>(java.util.List)",
2647+
"public java.util.List getArgs()",
2648+
"public java.lang.String toString()",
2649+
"public com.yahoo.search.grouping.request.FilterExpression copy()"
2650+
],
2651+
"fields" : [ ]
2652+
},
26392653
"com.yahoo.search.grouping.request.ArrayAtLookup" : {
26402654
"superClass" : "com.yahoo.search.grouping.request.DocumentValue",
26412655
"interfaces" : [ ],
@@ -3772,6 +3786,20 @@
37723786
],
37733787
"fields" : [ ]
37743788
},
3789+
"com.yahoo.search.grouping.request.NotPredicate" : {
3790+
"superClass" : "com.yahoo.search.grouping.request.FilterExpression",
3791+
"interfaces" : [ ],
3792+
"attributes" : [
3793+
"public"
3794+
],
3795+
"methods" : [
3796+
"public void <init>(com.yahoo.search.grouping.request.FilterExpression)",
3797+
"public com.yahoo.search.grouping.request.FilterExpression getExpression()",
3798+
"public java.lang.String toString()",
3799+
"public com.yahoo.search.grouping.request.FilterExpression copy()"
3800+
],
3801+
"fields" : [ ]
3802+
},
37753803
"com.yahoo.search.grouping.request.NowFunction" : {
37763804
"superClass" : "com.yahoo.search.grouping.request.FunctionNode",
37773805
"interfaces" : [ ],
@@ -3799,6 +3827,20 @@
37993827
],
38003828
"fields" : [ ]
38013829
},
3830+
"com.yahoo.search.grouping.request.OrPredicate" : {
3831+
"superClass" : "com.yahoo.search.grouping.request.FilterExpression",
3832+
"interfaces" : [ ],
3833+
"attributes" : [
3834+
"public"
3835+
],
3836+
"methods" : [
3837+
"public void <init>(java.util.List)",
3838+
"public java.util.List getArgs()",
3839+
"public java.lang.String toString()",
3840+
"public com.yahoo.search.grouping.request.FilterExpression copy()"
3841+
],
3842+
"fields" : [ ]
3843+
},
38023844
"com.yahoo.search.grouping.request.PredefinedFunction" : {
38033845
"superClass" : "com.yahoo.search.grouping.request.FunctionNode",
38043846
"interfaces" : [ ],

‎container-search/src/main/java/com/yahoo/search/grouping/request/AndPredicate.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
2+
package com.yahoo.search.grouping.request;
3+
4+
import com.yahoo.api.annotations.Beta;
5+
6+
import java.util.List;
7+
import java.util.stream.Collectors;
8+
9+
/**
10+
* Represents a logical conjunction (AND) of filter expressions used to match grouping elements.
11+
*
12+
* @author johsol
13+
*/
14+
@Beta
15+
public class AndPredicate extends FilterExpression {
16+
private final List<FilterExpression> args;
17+
18+
public AndPredicate(List<FilterExpression> args) {
19+
if (args == null || args.size() < 2) {
20+
throw new IllegalArgumentException("AndPredicate requires args to contain at least two elements.");
21+
}
22+
this.args = args;
23+
}
24+
25+
public List<FilterExpression> getArgs() {
26+
return args;
27+
}
28+
29+
@Override
30+
public String toString() {
31+
return "and(" + args.stream()
32+
.map(Object::toString)
33+
.collect(Collectors.joining(", ")) + ")";
34+
}
35+
36+
@Override
37+
public FilterExpression copy() {
38+
return new AndPredicate(args.stream().map(FilterExpression::copy).toList());
39+
}
40+
}

‎container-search/src/main/java/com/yahoo/search/grouping/request/NotPredicate.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
2+
package com.yahoo.search.grouping.request;
3+
4+
import com.yahoo.api.annotations.Beta;
5+
6+
import java.util.Objects;
7+
8+
/**
9+
* Represents a logical negation (NOT) of a filter expression used to exclude grouping elements.
10+
*
11+
* @author johsol
12+
*/
13+
@Beta
14+
public class NotPredicate extends FilterExpression {
15+
private final FilterExpression expression;
16+
17+
public NotPredicate(FilterExpression expression) {
18+
Objects.requireNonNull(expression, "Expression cannot be null");
19+
this.expression = expression;
20+
}
21+
22+
public FilterExpression getExpression() {
23+
return expression;
24+
}
25+
26+
@Override
27+
public String toString() {
28+
return "not(%s)".formatted(expression);
29+
}
30+
31+
@Override
32+
public FilterExpression copy() {
33+
return new NotPredicate(expression.copy());
34+
}
35+
}

‎container-search/src/main/java/com/yahoo/search/grouping/request/OrPredicate.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
2+
package com.yahoo.search.grouping.request;
3+
4+
import com.yahoo.api.annotations.Beta;
5+
6+
import java.util.List;
7+
import java.util.stream.Collectors;
8+
9+
/**
10+
* Represents a logical disjunction (OR) of filter expressions used to match grouping elements.
11+
*
12+
* @author johsol
13+
*/
14+
@Beta
15+
public class OrPredicate extends FilterExpression {
16+
private final List<FilterExpression> args;
17+
18+
public OrPredicate(List<FilterExpression> args) {
19+
if (args == null || args.size() < 2) {
20+
throw new IllegalArgumentException("OrPredicate requires args to contain at least two elements.");
21+
}
22+
this.args = args;
23+
}
24+
25+
public List<FilterExpression> getArgs() {
26+
return args;
27+
}
28+
29+
@Override
30+
public String toString() {
31+
return "or(" + args.stream()
32+
.map(Object::toString)
33+
.collect(Collectors.joining(", ")) + ")";
34+
}
35+
36+
@Override
37+
public FilterExpression copy() {
38+
return new OrPredicate(args.stream().map(FilterExpression::copy).toList());
39+
}
40+
}

‎container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.yahoo.search.grouping.request.AddFunction;
66
import com.yahoo.search.grouping.request.AggregatorNode;
77
import com.yahoo.search.grouping.request.AndFunction;
8+
import com.yahoo.search.grouping.request.AndPredicate;
89
import com.yahoo.search.grouping.request.ArrayAtLookup;
910
import com.yahoo.search.grouping.request.AttributeFunction;
1011
import com.yahoo.search.grouping.request.AttributeMapLookupValue;
@@ -61,9 +62,11 @@
6162
import com.yahoo.search.grouping.request.MonthOfYearFunction;
6263
import com.yahoo.search.grouping.request.MulFunction;
6364
import com.yahoo.search.grouping.request.NegFunction;
65+
import com.yahoo.search.grouping.request.NotPredicate;
6466
import com.yahoo.search.grouping.request.NormalizeSubjectFunction;
6567
import com.yahoo.search.grouping.request.NowFunction;
6668
import com.yahoo.search.grouping.request.OrFunction;
69+
import com.yahoo.search.grouping.request.OrPredicate;
6770
import com.yahoo.search.grouping.request.PredefinedFunction;
6871
import com.yahoo.search.grouping.request.RawValue;
6972
import com.yahoo.search.grouping.request.RegexPredicate;
@@ -103,6 +106,7 @@
103106
import com.yahoo.searchlib.expression.AddFunctionNode;
104107
import com.yahoo.searchlib.expression.AggregationRefNode;
105108
import com.yahoo.searchlib.expression.AndFunctionNode;
109+
import com.yahoo.searchlib.expression.AndPredicateNode;
106110
import com.yahoo.searchlib.expression.ArrayAtLookupNode;
107111
import com.yahoo.searchlib.expression.AttributeMapLookupNode;
108112
import com.yahoo.searchlib.expression.AttributeNode;
@@ -130,9 +134,11 @@
130134
import com.yahoo.searchlib.expression.MultiArgFunctionNode;
131135
import com.yahoo.searchlib.expression.MultiplyFunctionNode;
132136
import com.yahoo.searchlib.expression.NegateFunctionNode;
137+
import com.yahoo.searchlib.expression.NotPredicateNode;
133138
import com.yahoo.searchlib.expression.NormalizeSubjectFunctionNode;
134139
import com.yahoo.searchlib.expression.NumElemFunctionNode;
135140
import com.yahoo.searchlib.expression.OrFunctionNode;
141+
import com.yahoo.searchlib.expression.OrPredicateNode;
136142
import com.yahoo.searchlib.expression.RangeBucketPreDefFunctionNode;
137143
import com.yahoo.searchlib.expression.RawBucketResultNode;
138144
import com.yahoo.searchlib.expression.RawBucketResultNodeVector;
@@ -255,6 +261,14 @@ public AggregationResult toAggregationResult(GroupingExpression exp) {
255261
public FilterExpressionNode toFilterExpressionNode(FilterExpression expression) {
256262
if (expression instanceof RegexPredicate rp) {
257263
return new RegexPredicateNode(rp.getPattern(), toExpressionNode(rp.getExpression()));
264+
} else if (expression instanceof NotPredicate np) {
265+
return new NotPredicateNode(toFilterExpressionNode(np.getExpression()));
266+
} else if (expression instanceof OrPredicate op) {
267+
var args = op.getArgs().stream().map(this::toFilterExpressionNode).toList();
268+
return new OrPredicateNode(args);
269+
} else if (expression instanceof AndPredicate ap) {
270+
var args = ap.getArgs().stream().map(this::toFilterExpressionNode).toList();
271+
return new AndPredicateNode(args);
258272
} else {
259273
throw new IllegalInputException(
260274
"Can not convert '%s' to a filter expression.".formatted(expression.getClass().getSimpleName()));

‎container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ PARSER_BEGIN(GroupingParser)
2424

2525
package com.yahoo.search.grouping.request.parser;
2626

27+
import java.util.ArrayList;
2728
import java.util.List;
2829
import java.util.LinkedList;
2930
import com.yahoo.javacc.UnicodeUtilities;
@@ -131,6 +132,7 @@ TOKEN :
131132
<MOD: "mod"> |
132133
<MUL: "mul"> |
133134
<NEG: "neg"> |
135+
<NOT: "not"> |
134136
<NORMALIZESUBJECT: "normalizesubject"> |
135137
<NOW: "now"> |
136138
<OR: "or"> |
@@ -327,7 +329,10 @@ FilterExpression filterExp(GroupingOperation grp) :
327329
FilterExpression exp;
328330
}
329331
{
330-
exp = regexPredicate(grp)
332+
( (exp = regexPredicate(grp)) |
333+
(exp = notPredicate(grp)) |
334+
(exp = orPredicate(grp)) |
335+
(exp = andPredicate(grp)))
331336
{ return exp; }
332337
}
333338

@@ -955,6 +960,35 @@ RegexPredicate regexPredicate(GroupingOperation grp) :
955960
{ return new RegexPredicate(pattern, exp); }
956961
}
957962

963+
NotPredicate notPredicate(GroupingOperation grp) :
964+
{
965+
FilterExpression exp;
966+
}
967+
{
968+
<NOT> lbrace() exp = filterExp(grp) rbrace()
969+
{ return new NotPredicate(exp); }
970+
}
971+
972+
OrPredicate orPredicate(GroupingOperation grp) :
973+
{
974+
FilterExpression exp;
975+
List<FilterExpression> args = new ArrayList<FilterExpression>();
976+
}
977+
{
978+
<OR> lbrace() ( exp = filterExp(grp) { args.add(exp); } ( comma() exp = filterExp(grp) { args.add(exp); } )+ ) rbrace()
979+
{ return new OrPredicate(args); }
980+
}
981+
982+
AndPredicate andPredicate(GroupingOperation grp) :
983+
{
984+
FilterExpression exp;
985+
List<FilterExpression> args = new ArrayList<FilterExpression>();
986+
}
987+
{
988+
<AND> lbrace() ( exp = filterExp(grp) { args.add(exp); } ( comma() exp = filterExp(grp) { args.add(exp); } )+ ) rbrace()
989+
{ return new AndPredicate(args); }
990+
}
991+
958992
void bucket(GroupingOperation grp, BucketResolver resolver) :
959993
{
960994
ConstantValue from, to = null;
@@ -1068,6 +1102,7 @@ String identifier() :
10681102
<MOD> |
10691103
<MUL> |
10701104
<NEG> |
1105+
<NOT> |
10711106
<NORMALIZESUBJECT> |
10721107
<NOW> |
10731108
<OR> |

‎container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ void requireThatTokenImagesAreNotReservedWords() {
107107
"mod",
108108
"mul",
109109
"neg",
110+
"not",
110111
"normalizesubject",
111112
"now",
112113
"or",
@@ -637,6 +638,12 @@ void testFilter() {
637638
() -> assertParse(
638639
"all(group($myalias=foo) filter(regex(\".*mysubstring.*\", $myalias)) each(output(count())))",
639640
"all(group(foo) filter(regex(\".*mysubstring.*\", foo)) each(output(count())))"));
641+
642+
assertAll("filter with predicates",
643+
() -> assertParse("all(group(foo) filter(not(regex(\"mybar\", foo))) each(output(count())))"),
644+
() -> assertParse("all(group(foo) filter(or(regex(\"mybar\", foo), regex(\"mybaz\", foo), regex(\"myfoo\", boz))) each(output(count())))"),
645+
() -> assertParse("all(group(foo) filter(and(regex(\"mybar\", foo), regex(\"mybaz\", foo), regex(\"myfoo\", boz))) each(output(count())))"),
646+
() -> assertParse("all(group(foo) filter(and(or(regex(\"mybar\", foo), not(regex(\"mybaz\", foo))), regex(\"myfoo\", boz))) each(output(count())))"));
640647
}
641648

642649
// --------------------------------------------------------------------------------

‎container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717
import com.yahoo.searchlib.aggregation.HitsAggregationResult;
1818
import com.yahoo.searchlib.aggregation.SumAggregationResult;
1919
import com.yahoo.searchlib.expression.AddFunctionNode;
20+
import com.yahoo.searchlib.expression.AndPredicateNode;
2021
import com.yahoo.searchlib.expression.AttributeMapLookupNode;
2122
import com.yahoo.searchlib.expression.AttributeNode;
2223
import com.yahoo.searchlib.expression.ConstantNode;
2324
import com.yahoo.searchlib.expression.ExpressionNode;
2425
import com.yahoo.searchlib.expression.FilterExpressionNode;
26+
import com.yahoo.searchlib.expression.NotPredicateNode;
27+
import com.yahoo.searchlib.expression.OrPredicateNode;
2528
import com.yahoo.searchlib.expression.RegexPredicateNode;
2629
import com.yahoo.searchlib.expression.StrCatFunctionNode;
2730
import com.yahoo.searchlib.expression.StringResultNode;
@@ -33,6 +36,7 @@
3336
import java.util.LinkedList;
3437
import java.util.List;
3538
import java.util.TimeZone;
39+
import java.util.stream.Collectors;
3640

3741
import static org.junit.jupiter.api.Assertions.*;
3842

@@ -831,6 +835,18 @@ void require_that_filter_layout_is_correct() {
831835
"[[{ Attribute, filter = [Regex [Attribute]] }, { Attribute, result = [Count] }]]");
832836
}
833837

838+
@Test
839+
void require_that_filter_predicate_layout_is_correct() {
840+
assertLayout("all(group(a) filter(not(regex(\".*suffix$\", a))) each(output(count())))",
841+
"[[{ Attribute, filter = [Not [Regex [Attribute]]], result = [Count] }]]");
842+
assertLayout("all(group(a) filter(or(regex(\".*suffix$\", a),regex(\".*suffix$\", b))) each(output(count())))",
843+
"[[{ Attribute, filter = [Or [Regex [Attribute], Regex [Attribute]]], result = [Count] }]]");
844+
assertLayout("all(group(a) filter(and(regex(\".*suffix$\", a),regex(\".*suffix$\", b))) each(output(count())))",
845+
"[[{ Attribute, filter = [And [Regex [Attribute], Regex [Attribute]]], result = [Count] }]]");
846+
assertLayout("all(group(a) filter(and(or(regex(\".*suffix$\", a),regex(\".*suffix$\", b)), not(regex(\".*suffix$\", c)))) each(output(count())))",
847+
"[[{ Attribute, filter = [And [Or [Regex [Attribute], Regex [Attribute]], Not [Regex [Attribute]]]], result = [Count] }]]");
848+
}
849+
834850
private static void assertTotalGroupsAndSummaries(long expected, String query) {
835851
assertTotalGroupsAndSummaries(expected, Long.MAX_VALUE, query);
836852
}
@@ -1088,6 +1104,15 @@ private static String toSimpleName(FilterExpressionNode filterExp) {
10881104
if (filterExp instanceof RegexPredicateNode rpn) {
10891105
var simpleName = rpn.getExpression().map(LayoutWriter::toSimpleName).orElse("");
10901106
return "Regex [%s]".formatted(simpleName);
1107+
} else if (filterExp instanceof NotPredicateNode npn) {
1108+
var simpleName = npn.getExpression().map(LayoutWriter::toSimpleName).orElse("");
1109+
return "Not [%s]".formatted(simpleName);
1110+
} else if (filterExp instanceof OrPredicateNode opn) {
1111+
var simpleName = opn.getArgs().stream().map(LayoutWriter::toSimpleName).collect(Collectors.joining(", "));
1112+
return "Or [%s]".formatted(simpleName);
1113+
} else if (filterExp instanceof AndPredicateNode apn) {
1114+
var simpleName = apn.getArgs().stream().map(LayoutWriter::toSimpleName).collect(Collectors.joining(", "));
1115+
return "And [%s]".formatted(simpleName);
10911116
}
10921117
return filterExp.getClass().getSimpleName();
10931118
}

0 commit comments

Comments
 (0)