Skip to content

Commit b844e4a

Browse files
stroomdev66stroomdev66
authored
Merge pull request #4800 from gchq/gh-4791_having_equals
#4791 Fix numeric comparator in having clause
2 parents 25e0104 + ff423cd commit b844e4a

File tree

7 files changed

+132
-78
lines changed

7 files changed

+132
-78
lines changed

Diff for: ‎stroom-app/src/test/java/stroom/analytics/TestTableBuilderAnalytics.java

+14
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,20 @@ void testHavingCount() {
6868
basicTest(query, 9, 2);
6969
}
7070

71+
@Test
72+
void testHavingEquals() {
73+
final String query = """
74+
from index_view
75+
where UserId = user5
76+
eval count = count()
77+
eval EventTime = floorYear(EventTime)
78+
eval my_num = min(3)
79+
group by EventTime, UserId
80+
having count > 3 and my_num = 3
81+
select EventTime, UserId, count""";
82+
basicTest(query, 9, 2);
83+
}
84+
7185
@Test
7286
void testWindowCount() {
7387
final String query = """

Diff for: ‎stroom-app/src/test/java/stroom/search/TestInteractiveSearch2.java

+21
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,27 @@ void positiveCaseInsensitiveTest() {
9393
test(queryString, 5);
9494
}
9595

96+
/**
97+
* Test the having clause.
98+
*/
99+
@Test
100+
void testHavingEquals() {
101+
String queryString = """
102+
from "Test index"
103+
where UserId = user5 and Description = e0567
104+
and EventTime >= 2000-01-01T00:00:00.000Z
105+
and EventTime <= 2016-01-02T00:00:00.000Z
106+
eval my_num = 3
107+
having my_num = 3
108+
select
109+
StreamId as "Stream Id",
110+
EventId as "Event Id",
111+
EventTime as "Event Time",
112+
"annotation:Status" as Status
113+
""";
114+
test(queryString, 5);
115+
}
116+
96117
// @Test
97118
// void positiveCaseInsensitiveTestMultiComponent() {
98119
// final ExpressionOperator.Builder expression = buildExpression("UserId", "user5", "2000-01-01T00:00:00.000Z",

Diff for: ‎stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ExpressionPredicateFactory.java

+61-57
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ private <T> Optional<Predicate<T>> createDateTermPredicate(final ExpressionTerm
261261
private <T> Optional<Predicate<T>> createNumericTermPredicate(final ExpressionTerm term,
262262
final ValueFunctionFactory<T> valueFunctionFactory,
263263
final WordListProvider wordListProvider) {
264-
final Function<T, BigDecimal> numExtractor = valueFunctionFactory.createNumberExtractor();
264+
final Function<T, Double> numExtractor = valueFunctionFactory.createNumberExtractor();
265265
return switch (term.getCondition()) {
266266
case EQUALS -> NumericEquals.create(term, numExtractor);
267267
case NOT_EQUALS -> NotPredicate.create(NumericEquals.create(term, numExtractor));
@@ -281,10 +281,10 @@ private <T> Optional<Predicate<T>> createNumericTermPredicate(final ExpressionTe
281281
}
282282

283283

284-
private static BigDecimal getTermNumber(final ExpressionTerm term,
285-
final String value) {
284+
private static Double getTermNumber(final ExpressionTerm term,
285+
final String value) {
286286
try {
287-
return new BigDecimal(value);
287+
return new BigDecimal(value).doubleValue();
288288
} catch (final NumberFormatException e) {
289289
throw new MatchException(
290290
"Expected a numeric value for field \"" + term.getField() +
@@ -321,10 +321,10 @@ private static long getTermDate(final ExpressionTerm term,
321321
}
322322
}
323323

324-
private static BigDecimal[] getTermNumbers(final ExpressionTerm term,
325-
final Object value) {
324+
private static Double[] getTermNumbers(final ExpressionTerm term,
325+
final Object value) {
326326
final String[] values = value.toString().split(DELIMITER);
327-
final BigDecimal[] numbers = new BigDecimal[values.length];
327+
final Double[] numbers = new Double[values.length];
328328
for (int i = 0; i < values.length; i++) {
329329
numbers[i] = getTermNumber(term, values[i].trim());
330330
}
@@ -364,7 +364,7 @@ public interface ValueFunctionFactory<T> {
364364

365365
Function<T, Long> createDateExtractor();
366366

367-
Function<T, BigDecimal> createNumberExtractor();
367+
Function<T, Double> createNumberExtractor();
368368

369369
FieldType getFieldType();
370370
}
@@ -474,11 +474,11 @@ public boolean test(final T values) {
474474

475475
private abstract static class NumericExpressionTermPredicate<T> extends ExpressionTermPredicate<T> {
476476

477-
final BigDecimal termNum;
478-
final Function<T, BigDecimal> extractionFunction;
477+
final Double termNum;
478+
final Function<T, Double> extractionFunction;
479479

480480
private NumericExpressionTermPredicate(final ExpressionTerm term,
481-
final Function<T, BigDecimal> extractionFunction) {
481+
final Function<T, Double> extractionFunction) {
482482
super(term);
483483
termNum = getTermNumber(term, term.getValue());
484484
this.extractionFunction = extractionFunction;
@@ -496,123 +496,127 @@ private static <T> Optional<Predicate<T>> ifValue(final ExpressionTerm term,
496496
private static class NumericEquals<T> extends NumericExpressionTermPredicate<T> {
497497

498498
private NumericEquals(final ExpressionTerm term,
499-
final Function<T, BigDecimal> extractionFunction) {
499+
final Function<T, Double> extractionFunction) {
500500
super(term, extractionFunction);
501501
}
502502

503503
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
504-
final Function<T, BigDecimal> extractionFunction) {
504+
final Function<T, Double> extractionFunction) {
505505
return ifValue(term, () -> new NumericEquals<>(term, extractionFunction));
506506
}
507507

508508
@Override
509509
public boolean test(final T values) {
510-
final BigDecimal val = extractionFunction.apply(values);
511-
return Objects.equals(val, termNum);
510+
try {
511+
final Double val = extractionFunction.apply(values);
512+
return Objects.equals(val, termNum);
513+
} catch (final RuntimeException e) {
514+
return false;
515+
}
512516
}
513517
}
514518

515519
private static class NumericGreaterThan<T> extends NumericExpressionTermPredicate<T> {
516520

517521
private NumericGreaterThan(final ExpressionTerm term,
518-
final Function<T, BigDecimal> extractionFunction) {
522+
final Function<T, Double> extractionFunction) {
519523
super(term, extractionFunction);
520524
}
521525

522526
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
523-
final Function<T, BigDecimal> extractionFunction) {
527+
final Function<T, Double> extractionFunction) {
524528
return ifValue(term, () -> new NumericGreaterThan<>(term, extractionFunction));
525529
}
526530

527531
@Override
528532
public boolean test(final T values) {
529-
final BigDecimal val = extractionFunction.apply(values);
530-
int compVal = CompareUtil.compareBigDecimal(val, termNum);
533+
final Double val = extractionFunction.apply(values);
534+
int compVal = CompareUtil.compareDouble(val, termNum);
531535
return compVal > 0;
532536
}
533537
}
534538

535539
private static class NumericGreaterThanOrEqualTo<T> extends NumericExpressionTermPredicate<T> {
536540

537541
private NumericGreaterThanOrEqualTo(final ExpressionTerm term,
538-
final Function<T, BigDecimal> extractionFunction) {
542+
final Function<T, Double> extractionFunction) {
539543
super(term, extractionFunction);
540544
}
541545

542546
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
543-
final Function<T, BigDecimal> extractionFunction) {
547+
final Function<T, Double> extractionFunction) {
544548
return ifValue(term, () -> new NumericGreaterThanOrEqualTo<>(term, extractionFunction));
545549
}
546550

547551
@Override
548552
public boolean test(final T values) {
549-
final BigDecimal val = extractionFunction.apply(values);
550-
int compVal = CompareUtil.compareBigDecimal(val, termNum);
553+
final Double val = extractionFunction.apply(values);
554+
int compVal = CompareUtil.compareDouble(val, termNum);
551555
return compVal >= 0;
552556
}
553557
}
554558

555559
private static class NumericLessThan<T> extends NumericExpressionTermPredicate<T> {
556560

557561
private NumericLessThan(final ExpressionTerm term,
558-
final Function<T, BigDecimal> extractionFunction) {
562+
final Function<T, Double> extractionFunction) {
559563
super(term, extractionFunction);
560564
}
561565

562566
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
563-
final Function<T, BigDecimal> extractionFunction) {
567+
final Function<T, Double> extractionFunction) {
564568
return ifValue(term, () -> new NumericLessThan<>(term, extractionFunction));
565569
}
566570

567571
@Override
568572
public boolean test(final T values) {
569-
final BigDecimal val = extractionFunction.apply(values);
570-
int compVal = CompareUtil.compareBigDecimal(val, termNum);
573+
final Double val = extractionFunction.apply(values);
574+
int compVal = CompareUtil.compareDouble(val, termNum);
571575
return compVal < 0;
572576
}
573577
}
574578

575579
private static class NumericLessThanOrEqualTo<T> extends NumericExpressionTermPredicate<T> {
576580

577581
private NumericLessThanOrEqualTo(final ExpressionTerm term,
578-
final Function<T, BigDecimal> extractionFunction) {
582+
final Function<T, Double> extractionFunction) {
579583
super(term, extractionFunction);
580584
}
581585

582586
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
583-
final Function<T, BigDecimal> extractionFunction) {
587+
final Function<T, Double> extractionFunction) {
584588
return ifValue(term, () -> new NumericLessThanOrEqualTo<>(term, extractionFunction));
585589
}
586590

587591
@Override
588592
public boolean test(final T values) {
589-
final BigDecimal val = extractionFunction.apply(values);
590-
int compVal = CompareUtil.compareBigDecimal(val, termNum);
593+
final Double val = extractionFunction.apply(values);
594+
int compVal = CompareUtil.compareDouble(val, termNum);
591595
return compVal <= 0;
592596
}
593597
}
594598

595599
private static class NumericBetween<T> extends ExpressionTermPredicate<T> {
596600

597-
private final BigDecimal[] between;
598-
private final Function<T, BigDecimal> extractionFunction;
601+
private final Double[] between;
602+
private final Function<T, Double> extractionFunction;
599603

600604
private NumericBetween(final ExpressionTerm term,
601-
final Function<T, BigDecimal> extractionFunction,
602-
final BigDecimal[] between) {
605+
final Function<T, Double> extractionFunction,
606+
final Double[] between) {
603607
super(term);
604608
this.between = between;
605609
this.extractionFunction = extractionFunction;
606610
}
607611

608612
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
609-
final Function<T, BigDecimal> extractionFunction) {
613+
final Function<T, Double> extractionFunction) {
610614
return ifValue(term, () -> {
611-
final BigDecimal[] between = getTermNumbers(term, term.getValue());
615+
final Double[] between = getTermNumbers(term, term.getValue());
612616
if (between.length != 2) {
613617
throw new MatchException("2 numbers needed for between query");
614618
}
615-
if (CompareUtil.compareBigDecimal(between[0], between[1]) >= 0) {
619+
if (CompareUtil.compareDouble(between[0], between[1]) >= 0) {
616620
throw new MatchException("From number must be lower than to number");
617621
}
618622
return new NumericBetween<>(term, extractionFunction, between);
@@ -621,28 +625,28 @@ private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
621625

622626
@Override
623627
public boolean test(final T values) {
624-
final BigDecimal val = extractionFunction.apply(values);
625-
return CompareUtil.compareBigDecimal(val, between[0]) >= 0
626-
&& CompareUtil.compareBigDecimal(val, between[1]) <= 0;
628+
final Double val = extractionFunction.apply(values);
629+
return CompareUtil.compareDouble(val, between[0]) >= 0
630+
&& CompareUtil.compareDouble(val, between[1]) <= 0;
627631
}
628632
}
629633

630634
private static class NumericIn<T> extends ExpressionTermPredicate<T> {
631635

632-
private final BigDecimal[] in;
633-
private final Function<T, BigDecimal> extractionFunction;
636+
private final Double[] in;
637+
private final Function<T, Double> extractionFunction;
634638

635639
private NumericIn(final ExpressionTerm term,
636-
final Function<T, BigDecimal> extractionFunction,
637-
final BigDecimal[] in) {
640+
final Function<T, Double> extractionFunction,
641+
final Double[] in) {
638642
super(term);
639643
this.in = in;
640644
this.extractionFunction = extractionFunction;
641645
}
642646

643647
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
644-
final Function<T, BigDecimal> extractionFunction) {
645-
final BigDecimal[] in = getTermNumbers(term, term.getValue());
648+
final Function<T, Double> extractionFunction) {
649+
final Double[] in = getTermNumbers(term, term.getValue());
646650
// If there are no terms then always a false match.
647651
if (in.length == 0) {
648652
return Optional.of(values -> false);
@@ -652,8 +656,8 @@ private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
652656

653657
@Override
654658
public boolean test(final T values) {
655-
final BigDecimal val = extractionFunction.apply(values);
656-
for (final BigDecimal n : in) {
659+
final Double val = extractionFunction.apply(values);
660+
for (final Double n : in) {
657661
if (Objects.equals(n, val)) {
658662
return true;
659663
}
@@ -664,19 +668,19 @@ public boolean test(final T values) {
664668

665669
private static class NumericInDictionary<T> extends ExpressionTermPredicate<T> {
666670

667-
private final Function<T, BigDecimal> extractionFunction;
668-
private final BigDecimal[] in;
671+
private final Function<T, Double> extractionFunction;
672+
private final Double[] in;
669673

670674
private NumericInDictionary(final ExpressionTerm term,
671-
final Function<T, BigDecimal> extractionFunction,
672-
final BigDecimal[] in) {
675+
final Function<T, Double> extractionFunction,
676+
final Double[] in) {
673677
super(term);
674678
this.extractionFunction = extractionFunction;
675679
this.in = in;
676680
}
677681

678682
private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
679-
final Function<T, BigDecimal> extractionFunction,
683+
final Function<T, Double> extractionFunction,
680684
final WordListProvider wordListProvider) {
681685
final String[] words;
682686
if (term.getDocRef() != null) {
@@ -688,7 +692,7 @@ private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
688692
return Optional.of(values -> false);
689693
}
690694

691-
final BigDecimal[] in = new BigDecimal[words.length];
695+
final Double[] in = new Double[words.length];
692696
for (int i = 0; i < words.length; i++) {
693697
final String word = words[i];
694698
in[i] = getTermNumber(term, word);
@@ -698,8 +702,8 @@ private static <T> Optional<Predicate<T>> create(final ExpressionTerm term,
698702

699703
@Override
700704
public boolean test(final T values) {
701-
final BigDecimal val = extractionFunction.apply(values);
702-
for (final BigDecimal n : in) {
705+
final Double val = extractionFunction.apply(values);
706+
for (final Double n : in) {
703707
if (Objects.equals(n, val)) {
704708
return true;
705709
}

Diff for: ‎stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/StringValueFunctionFactory.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,14 @@ public Function<String, Long> createDateExtractor() {
3333
}
3434

3535
@Override
36-
public Function<String, BigDecimal> createNumberExtractor() {
37-
return BigDecimal::new;
36+
public Function<String, Double> createNumberExtractor() {
37+
return string -> {
38+
try {
39+
return new BigDecimal(string).doubleValue();
40+
} catch (final NumberFormatException e) {
41+
return null;
42+
}
43+
};
3844
}
3945

4046
@Override

0 commit comments

Comments
 (0)