Merge pull request #4794 from gchq/gh-4552_dynamic_fields_filter_merge

Gh 4552 dynamic fields filter merge

Author: stroomdev66

Diff for: stroom-activity/stroom-activity-api/src/main/java/stroom/activity/api/ActivityService.java

@@ -18,7 +18,7 @@
 
 import stroom.activity.shared.Activity;
 import stroom.activity.shared.ActivityValidationResult;
-import stroom.util.shared.QuickFilterResultPage;
+import stroom.util.shared.ResultPage;
 import stroom.util.shared.UserRef;
 import stroom.util.shared.filter.FilterFieldDefinition;
 
@@ -36,7 +36,7 @@ public interface ActivityService {
 
     int deleteAllByOwner(UserRef ownerRef);
 
-    QuickFilterResultPage<Activity> find(final String filter);
+    ResultPage<Activity> find(final String filter);
 
     List<FilterFieldDefinition> listFieldDefinitions();
 

Diff for: stroom-activity/stroom-activity-impl-db/build.gradle

@@ -7,6 +7,9 @@ dependencies {
     implementation project(':stroom-config:stroom-config-common')
     implementation project(':stroom-core-shared')
     implementation project(':stroom-db-util')
+    implementation project(':stroom-query:stroom-query-api')
+    implementation project(':stroom-query:stroom-query-language-api')
+    implementation project(':stroom-query:stroom-query-common')
     implementation project(':stroom-util')
     implementation project(':stroom-util-shared')
 

Diff for: stroom-activity/stroom-activity-impl-db/src/main/java/stroom/activity/impl/db/ActivityDaoImpl.java

@@ -38,7 +38,6 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static stroom.activity.impl.db.jooq.tables.Activity.ACTIVITY;
@@ -161,7 +160,7 @@ public List<Activity> find(final FindActivityCriteria criteria) {
                         .fetch())
                 .stream()
                 .map(RECORD_TO_ACTIVITY_MAPPER)
-                .collect(Collectors.toList());
+                .toList();
     }
 
     @Override
@@ -187,7 +186,7 @@ public List<Activity> find(
                 return streamFunction.apply(activityStream)
                         .skip(offset)
                         .limit(limit)
-                        .collect(Collectors.toList());
+                        .toList();
             }
         });
     }

Diff for: stroom-activity/stroom-activity-impl-db/src/test/java/stroom/activity/impl/db/TestActivityServiceImpl.java

@@ -24,6 +24,7 @@
 import stroom.activity.shared.Activity;
 import stroom.activity.shared.Activity.Prop;
 import stroom.activity.shared.ActivityValidationResult;
+import stroom.query.common.v2.ExpressionPredicateFactory;
 import stroom.security.api.SecurityContext;
 import stroom.security.mock.MockSecurityContext;
 import stroom.test.common.util.db.DbTestUtil;
@@ -54,9 +55,10 @@ class TestActivityServiceImpl {
     void before() {
         final ActivityDbConnProvider activityDbConnProvider = DbTestUtil.getTestDbDatasource(
                 new ActivityDbModule(), new ActivityDbConfig());
+        final ExpressionPredicateFactory expressionPredicateFactory = new ExpressionPredicateFactory();
 
         final ActivityDao activityDao = new ActivityDaoImpl(activityDbConnProvider);
-        activityService = new ActivityServiceImpl(securityContext, activityDao);
+        activityService = new ActivityServiceImpl(securityContext, activityDao, expressionPredicateFactory);
     }
 
     @Test

Diff for: stroom-activity/stroom-activity-impl/build.gradle

@@ -5,6 +5,9 @@ dependencies {
     implementation project(':stroom-core-shared')
     implementation project(':stroom-event-logging:stroom-event-logging-api')
     implementation project(':stroom-event-logging-rs:stroom-event-logging-rs-api')
+    implementation project(':stroom-query:stroom-query-api')
+    implementation project(':stroom-query:stroom-query-language-api')
+    implementation project(':stroom-query:stroom-query-common')
     implementation project(':stroom-security:stroom-security-api')
     implementation project(':stroom-util-shared')
     implementation project(':stroom-util')

Diff for: stroom-activity/stroom-activity-impl/src/main/java/stroom/activity/impl/ActivityResourceImpl.java

@@ -12,7 +12,7 @@
 import stroom.event.logging.rs.api.AutoLogged;
 import stroom.event.logging.rs.api.AutoLogged.OperationType;
 import stroom.util.rest.RestUtil;
-import stroom.util.shared.QuickFilterResultPage;
+import stroom.util.shared.ResultPage;
 import stroom.util.shared.filter.FilterFieldDefinition;
 
 import com.google.common.base.Strings;
@@ -50,7 +50,7 @@ class ActivityResourceImpl implements ActivityResource {
 
     @AutoLogged(value = OperationType.MANUALLY_LOGGED)
     @Override
-    public QuickFilterResultPage<Activity> list(final String filter) {
+    public ResultPage<Activity> list(final String filter) {
         final boolean loggingRequired = !Strings.isNullOrEmpty(filter);
         ThreadLocalLogState.setLogged(!loggingRequired);
 
@@ -66,10 +66,10 @@ public QuickFilterResultPage<Activity> list(final String filter) {
                         .build())
                 .withComplexLoggedResult(searchEventAction -> {
                     // Do the work
-                    final QuickFilterResultPage<Activity> result = activityServiceProvider.get().find(filter);
+                    final ResultPage<Activity> result = activityServiceProvider.get().find(filter);
 
                     final SearchEventAction newSearchEventAction = searchEventAction.newCopyBuilder()
-                            .withQuery(buildRawQuery(result.getQualifiedFilterInput()))
+                            .withQuery(buildRawQuery(filter))
                             .withResultPage(StroomEventLoggingUtil.createResultPage(result))
                             .withTotalResults(BigInteger.valueOf(result.size()))
                             .build();

Diff for: stroom-activity/stroom-activity-impl/src/main/java/stroom/activity/impl/ActivityServiceImpl.java

@@ -20,31 +20,35 @@
 import stroom.activity.api.FindActivityCriteria;
 import stroom.activity.shared.Activity;
 import stroom.activity.shared.ActivityValidationResult;
+import stroom.expression.api.DateTimeSettings;
+import stroom.query.common.v2.ExpressionPredicateFactory;
+import stroom.query.common.v2.ExpressionPredicateFactory.ValueFunctionFactories;
+import stroom.query.common.v2.FieldProviderImpl;
+import stroom.query.common.v2.SimpleStringExpressionParser.FieldProvider;
+import stroom.query.common.v2.ValueFunctionFactoriesImpl;
 import stroom.security.api.SecurityContext;
 import stroom.security.shared.AppPermission;
 import stroom.util.AuditUtil;
-import stroom.util.filter.FilterFieldMapper;
-import stroom.util.filter.FilterFieldMappers;
-import stroom.util.filter.QuickFilterPredicateFactory;
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
 import stroom.util.shared.EntityServiceException;
 import stroom.util.shared.PermissionException;
-import stroom.util.shared.QuickFilterResultPage;
+import stroom.util.shared.ResultPage;
 import stroom.util.shared.UserRef;
 import stroom.util.shared.filter.FilterFieldDefinition;
 
 import com.google.common.base.Strings;
 import jakarta.inject.Inject;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class ActivityServiceImpl implements ActivityService {
@@ -53,12 +57,15 @@ public class ActivityServiceImpl implements ActivityService {
 
     private final SecurityContext securityContext;
     private final ActivityDao dao;
+    private final ExpressionPredicateFactory expressionPredicateFactory;
 
     @Inject
     public ActivityServiceImpl(final SecurityContext securityContext,
-                               final ActivityDao dao) {
+                               final ActivityDao dao,
+                               final ExpressionPredicateFactory expressionPredicateFactory) {
         this.securityContext = securityContext;
         this.dao = dao;
+        this.expressionPredicateFactory = expressionPredicateFactory;
     }
 
     @Override
@@ -132,41 +139,35 @@ public int deleteAllByOwner(final UserRef ownerRef) {
     }
 
     @Override
-    public QuickFilterResultPage<Activity> find(final String filter) {
+    public ResultPage<Activity> find(final String filter) {
         return securityContext.secureResult(() -> {
             // We have to deser all the activities to be able to search them but hopefully
             // there are not that many to worry about
             final List<Activity> allActivities = getAllUserActivities();
 
             final List<Activity> filteredActivities;
-            final String fullyQualifiedInput;
             if (!Strings.isNullOrEmpty(filter)) {
-
                 final List<FilterFieldDefinition> fieldDefinitions = buildFieldDefinitions(allActivities);
-
-                final FilterFieldMappers<Activity> fieldMappers = buildFieldMappers(fieldDefinitions);
-
-                filteredActivities = QuickFilterPredicateFactory.filterStream(
-                                filter, fieldMappers, allActivities.stream())
-                        .collect(Collectors.toList());
-
-                fullyQualifiedInput = QuickFilterPredicateFactory.fullyQualifyInput(filter, fieldMappers);
+                final FieldProvider fieldProvider = new FieldProviderImpl(fieldDefinitions);
+                final ValueFunctionFactories<Activity> valueFunctionFactories =
+                        buildValueFunctionFactories(fieldDefinitions);
+                filteredActivities = expressionPredicateFactory.filterAndSortStream(
+                                allActivities.stream(),
+                                filter, fieldProvider, valueFunctionFactories,
+                                Optional.of(Comparator.comparingInt((Activity activity) -> activity.getId())))
+                        .toList();
             } else {
                 filteredActivities = allActivities;
-                fullyQualifiedInput = filter;
             }
 
-            return QuickFilterResultPage.createCriterialBasedList(
+            return ResultPage.createCriterialBasedList(
                     filteredActivities,
-                    new FindActivityCriteria(),
-                    filteredActivities.size(),
-                    fullyQualifiedInput);
+                    new FindActivityCriteria());
         });
     }
 
     @Override
     public List<FilterFieldDefinition> listFieldDefinitions() {
-
         return securityContext.secureResult(() -> {
             final List<Activity> allActivities = getAllUserActivities();
 
@@ -191,7 +192,7 @@ private List<FilterFieldDefinition> buildFieldDefinitions(final List<Activity> a
                     }
                 })
                 .distinct()
-                .collect(Collectors.toList());
+                .toList();
     }
 
     private List<Activity> getAllUserActivities() {
@@ -240,26 +241,22 @@ public ActivityValidationResult validate(final Activity activity) {
         return new ActivityValidationResult(valid, String.join("\n", messages));
     }
 
-    private FilterFieldMappers<Activity> buildFieldMappers(final List<FilterFieldDefinition> fieldDefinitions) {
+    private ValueFunctionFactories<Activity> buildValueFunctionFactories(
+            final List<FilterFieldDefinition> fieldDefinitions) {
         // Extracting the value out of the json details is not very efficient.  May be better to use
         // something like jsoniter on the raw json.
-        final FilterFieldMappers<Activity> fieldMappers = FilterFieldMappers.of(fieldDefinitions.stream()
-                .map(fieldDefinition ->
-                        FilterFieldMapper.of(fieldDefinition, (Activity activity) -> {
-
-                            // Use the field displayname to look up the matching prop in the activity details
-                            final String value = Optional.ofNullable(activity)
-                                    .flatMap(activity2 ->
-                                            Optional.ofNullable(activity2.getDetails()))
-                                    .map(details ->
-                                            details.valueByName(fieldDefinition.getDisplayName()))
-                                    .orElse(null);
-
-                            LOGGER.trace("FilterFieldDefinition: {}, value {}",
-                                    fieldDefinition, value);
-                            return value;
-                        }))
-                .collect(Collectors.toList()));
-        return fieldMappers;
+        final ValueFunctionFactoriesImpl valueFunctionFactories = new ValueFunctionFactoriesImpl();
+        fieldDefinitions.stream().forEach(fieldDefinition -> {
+            final Function<Activity, String> function = activity -> {
+                return Optional.ofNullable(activity)
+                        .flatMap(activity2 ->
+                                Optional.ofNullable(activity2.getDetails()))
+                        .map(details ->
+                                details.valueByName(fieldDefinition.getDisplayName()))
+                        .orElse(null);
+            };
+            valueFunctionFactories.put(fieldDefinition, function);
+        });
+        return valueFunctionFactories;
     }
 }

Diff for: stroom-activity/stroom-activity-impl/src/test/resources/logback-test.xml

@@ -0,0 +1,19 @@
+<configuration>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="stroom" level="debug" />
+
+    <!--<logger name="org.flywaydb.core.internal.sqlscript" level="debug" />-->
+
+    <root level="error">
+        <appender-ref ref="STDOUT" />
+    </root>
+
+</configuration>

Diff for: stroom-analytics/stroom-analytics-impl-db/src/main/java/stroom/analytics/impl/db/ExecutionScheduleDaoImpl.java

@@ -11,7 +11,6 @@
 import stroom.analytics.shared.ExecutionTracker;
 import stroom.analytics.shared.ScheduleBounds;
 import stroom.db.util.JooqUtil;
-import stroom.db.util.StringMatchConditionUtil;
 import stroom.docref.DocRef;
 import stroom.security.api.SecurityContext;
 import stroom.security.shared.AppPermission;
@@ -64,8 +63,8 @@ public ResultPage<ExecutionSchedule> fetchExecutionSchedule(final ExecutionSched
                 Optional.ofNullable(request.getOwnerDocRef())
                         .map(DocRef::getUuid)
                         .map(EXECUTION_SCHEDULE.DOC_UUID::eq),
-                Optional.ofNullable(StringMatchConditionUtil
-                        .getCondition(EXECUTION_SCHEDULE.NODE_NAME, request.getNodeName())),
+                Optional.ofNullable(request.getNodeName())
+                        .map(EXECUTION_SCHEDULE.NODE_NAME::eq),
                 Optional.ofNullable(request.getEnabled())
                         .map(EXECUTION_SCHEDULE.ENABLED::eq));
         final Collection<OrderField<?>> orderFields = createExecutionScheduleOrderFields(request);

Diff for: stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AbstractScheduledQueryExecutor.java

@@ -24,7 +24,6 @@
 import stroom.analytics.shared.ExecutionTracker;
 import stroom.analytics.shared.ScheduleBounds;
 import stroom.docref.DocRef;
-import stroom.docref.StringMatch;
 import stroom.docrefinfo.api.DocRefInfoService;
 import stroom.node.api.NodeInfo;
 import stroom.security.api.SecurityContext;
@@ -162,7 +161,7 @@ private void execRule(final T doc,
                 .builder()
                 .ownerDocRef(docRef)
                 .enabled(true)
-                .nodeName(StringMatch.equals(nodeInfo.getThisNodeName(), true))
+                .nodeName(nodeInfo.getThisNodeName())
                 .build();
 
         final ResultPage<ExecutionSchedule> executionSchedules = executionScheduleDao.fetchExecutionSchedule(request);

Diff for: stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticsSearchProvider.java

@@ -29,7 +29,7 @@
 import stroom.query.common.v2.CoprocessorsFactory;
 import stroom.query.common.v2.CoprocessorsImpl;
 import stroom.query.common.v2.DataStoreSettings;
-import stroom.query.common.v2.FieldInfoResultPageBuilder;
+import stroom.query.common.v2.FieldInfoResultPageFactory;
 import stroom.query.common.v2.ResultStore;
 import stroom.query.common.v2.ResultStoreFactory;
 import stroom.query.common.v2.SearchProvider;
@@ -51,26 +51,24 @@ public class AnalyticsSearchProvider implements SearchProvider, IsSpecialExplore
     private final ResultStoreFactory resultStoreFactory;
     private final FederatedSearchExecutor federatedSearchExecutor;
     private final AnalyticsNodeSearchTaskCreator nodeSearchTaskCreator;
+    private final FieldInfoResultPageFactory fieldInfoResultPageFactory;
 
     @Inject
     public AnalyticsSearchProvider(final CoprocessorsFactory coprocessorsFactory,
                                    final ResultStoreFactory resultStoreFactory,
                                    final FederatedSearchExecutor federatedSearchExecutor,
-                                   final AnalyticsNodeSearchTaskCreator nodeSearchTaskCreator) {
+                                   final AnalyticsNodeSearchTaskCreator nodeSearchTaskCreator,
+                                   final FieldInfoResultPageFactory fieldInfoResultPageFactory) {
         this.coprocessorsFactory = coprocessorsFactory;
         this.resultStoreFactory = resultStoreFactory;
         this.federatedSearchExecutor = federatedSearchExecutor;
         this.nodeSearchTaskCreator = nodeSearchTaskCreator;
+        this.fieldInfoResultPageFactory = fieldInfoResultPageFactory;
     }
 
     @Override
     public ResultPage<QueryField> getFieldInfo(final FindFieldCriteria criteria) {
-//        if (!getType().equals(criteria.getDataSourceRef().getType())) {
-//            return ResultPage.empty();
-//        }
-        return FieldInfoResultPageBuilder.builder(criteria)
-                .addAll(AnalyticFields.getFields())
-                .build();
+        return fieldInfoResultPageFactory.create(criteria, AnalyticFields.getFields());
     }
 
     @Override

Diff for: stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/Detection.java

@@ -15,7 +15,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
-import java.util.stream.Collectors;
 
 @JsonPropertyOrder({
         "detectTime",
@@ -444,7 +443,7 @@ public Builder withValues(final Map<String, String> values) {
                     .stream()
                     .filter(entry -> entry.getKey() != null)
                     .map(entry -> new DetectionValue(entry.getKey(), entry.getValue()))
-                    .collect(Collectors.toList());
+                    .toList();
             return this;
         }