Migrate tag option filtering from slicing to MoreSearchAdapter

changelog/unreleased/pr-25896.toml

@@ -1,4 +1,4 @@
 type = "a"
 message = "Ability to assign tags to event definitions."
 
-pulls = ["25896", "25940", "26079"]
+pulls = ["25896", "25940", "26079", "26841"]

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java

@@ -18,6 +18,7 @@
 
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Streams;
+import jakarta.annotation.Nullable;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import org.graylog.events.event.EventDto;
@@ -330,10 +331,14 @@ public List<Slice> aggregateSlices(String queryString, TimeRange timerange, Set<
     @Override
     public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timerange, Set<String> affectedIndices,
                                                 Set<String> eventStreams, String filterString, SourceStreamFilter sourceStreamFilter,
-                                                Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, int maxBuckets) {
+                                                Map<String, Set<String>> extraFilters, String slicingColumn, @Nullable String bucketPattern,
+                                                Map<String, Object> meta, int maxBuckets) {
         final var builder = AggregationBuilders.terms(slicesAggregationName)
                 .field(slicingColumn)
                 .size(maxBuckets);
+        if (bucketPattern != null) {
+            builder.includeExclude(new IncludeExclude(bucketPattern, null));
+        }
 
         return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder);
     }

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java

@@ -18,6 +18,7 @@
 
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Streams;
+import jakarta.annotation.Nullable;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import org.graylog.events.event.EventDto;
@@ -49,6 +50,7 @@
 import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.ParsedTerms;
+import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.graylog.shaded.opensearch2.org.opensearch.search.builder.SearchSourceBuilder;
 import org.graylog.shaded.opensearch2.org.opensearch.search.sort.FieldSortBuilder;
 import org.graylog.shaded.opensearch2.org.opensearch.search.sort.SortOrder;
@@ -332,10 +334,14 @@ private List<Slice> aggregateSlices(String queryString, TimeRange timerange, Set
     @Override
     public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timerange, Set<String> affectedIndices,
                                              Set<String> eventStreams, String filterString, SourceStreamFilter sourceStreamFilter,
-                                             Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, int maxBuckets) {
-        AggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName)
+                                             Map<String, Set<String>> extraFilters, String slicingColumn, @Nullable String bucketPattern,
+                                             Map<String, Object> meta, int maxBuckets) {
+        TermsAggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName)
                         .field(slicingColumn)
                         .size(maxBuckets);
+        if (bucketPattern != null) {
+            builder.includeExclude(new IncludeExclude(bucketPattern, null));
+        }
 
         return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder);
     }

graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java

@@ -17,6 +17,7 @@
 package org.graylog.storage.opensearch3;
 
 import com.google.common.base.Stopwatch;
+import jakarta.annotation.Nullable;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import org.graylog.events.event.EventDto;
@@ -343,10 +344,17 @@ private org.opensearch.client.opensearch._types.SortOrder sortOrder(Sorting sort
     @Override
     public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timerange, Set<String> affectedIndices,
                                                 Set<String> eventStreams, String filterString, SourceStreamFilter sourceStreamFilter,
-                                                Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, int maxBuckets) {
+                                                Map<String, Set<String>> extraFilters, String slicingColumn, @Nullable String bucketPattern,
+                                                Map<String, Object> meta, int maxBuckets) {
         final var filter = createQuery(queryString, timerange, eventStreams, filterString, sourceStreamFilter, extraFilters);
         final var aggregation = Aggregation.builder()
-                .terms(terms -> terms.field(slicingColumn).size(maxBuckets))
+                .terms(terms -> {
+                    terms.field(slicingColumn).size(maxBuckets);
+                    if (bucketPattern != null) {
+                        terms.include(inc -> inc.regexp(bucketPattern));
+                    }
+                    return terms;
+                })
                 .build();
 
         final var searchResult = executeAggregation(filter, affectedIndices, SLICES_AGGREGATION_NAME, aggregation);

graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java

@@ -29,6 +29,8 @@
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.MediaType;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
+import org.graylog.events.search.EventsFilterOptions;
+import org.graylog.events.search.EventsFilterOptionsRequest;
 import org.graylog.events.search.EventsHistogramResult;
 import org.graylog.events.search.EventsSearchParameters;
 import org.graylog.events.search.EventsSearchResult;
@@ -88,6 +90,14 @@ public Slices slices(@Context SearchUser searchUser, @Parameter(name = "JSON bod
         return sliceService.slices(firstNonNull(request, EventsSlicesRequest.empty()), getSubject(), searchUser);
     }
 
+    @POST
+    @Path("/filter_options")
+    @Operation(summary = "Get the available values for the given event fields")
+    @NoAuditEvent("Doesn't change any data, only collects filter values")
+    public EventsFilterOptions filterOptions(@Parameter(name = "JSON body") final EventsFilterOptionsRequest request) {
+        return searchService.filterOptions(firstNonNull(request, EventsFilterOptionsRequest.empty()), getSubject());
+    }
+
     @POST
     @Path("/histogram")
     @Operation(summary = "Build histogram of events over time")

graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.events.search;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog.events.event.EventDto;
+
+import java.util.List;
+
+/**
+ * Values available to filter the events table by. Fields that were not requested are omitted.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public record EventsFilterOptions(@JsonProperty(EventDto.FIELD_TAGS) List<String> tags) {
+}

graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.events.search;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
+import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
+import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+
+import java.util.List;
+
+/**
+ * Request for the values available to filter the events table by.
+ *
+ * @param fields     the event fields to return available values for
+ * @param query      optional query to narrow the events the values are collected from, e.g. to scope
+ *                   the result to a single source stream
+ * @param fieldQuery optional search text the returned values must contain (case-insensitive)
+ * @param timerange  the time range to collect values from, defaulting to the last 30 days
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record EventsFilterOptionsRequest(@JsonProperty("fields") List<String> fields,
+                                         @JsonProperty("query") String query,
+                                         @JsonProperty("field_query") String fieldQuery,
+                                         @JsonProperty("timerange") TimeRange timerange) {
+
+    private static final int DEFAULT_RANGE_SECONDS = 30 * 24 * 60 * 60;
+
+    // Defensive cap: field values are short (tags max out at 128 chars), so longer search text can
+    // never match and would only bloat the pattern sent to the search backend.
+    private static final int MAX_FIELD_QUERY_LENGTH = 256;
+
+    public EventsFilterOptionsRequest {
+        fields = fields == null ? List.of() : fields;
+        query = query == null ? "" : query;
+        fieldQuery = fieldQuery == null ? "" : fieldQuery.substring(0, Math.min(fieldQuery.length(), MAX_FIELD_QUERY_LENGTH));
+        timerange = timerange == null ? defaultTimerange() : timerange;
+    }
+
+    public static EventsFilterOptionsRequest empty() {
+        return new EventsFilterOptionsRequest(List.of(), "", "", null);
+    }
+
+    private static TimeRange defaultTimerange() {
+        try {
+            return RelativeRange.create(DEFAULT_RANGE_SECONDS);
+        } catch (InvalidRangeParametersException e) {
+            throw new IllegalStateException("Unable to create the default filter options time range", e);
+        }
+    }
+}

graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java

@@ -25,17 +25,24 @@
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
+import org.graylog2.rest.resources.entities.Slice;
 import org.graylog2.streams.StreamService;
 
 import java.time.ZoneId;
 import java.util.Collection;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.graylog.events.search.EventsSearchFilter.NULL_VALUE;
 
 public class EventsSearchService extends AbstractEventsSearchService {
+    // Upper bound on the values returned per filter option field. The dropdown shows the most-used
+    // values first and relies on the field query for anything beyond the cap.
+    private static final int MAX_FILTER_OPTIONS = 50;
+
     private final MoreSearch moreSearch;
     private final StreamService streamService;
 
@@ -95,6 +102,56 @@ public EventsHistogramResult histogram(EventsSearchParameters parameters, Subjec
         return EventsHistogramResult.fromResult(result);
     }
 
+    /**
+     * Returns the values the events table can be filtered by, collected from the events the subject is
+     * permitted to see. Deliberately aggregates through {@link MoreSearch} rather than the views search
+     * engine so that the source stream permissions of the subject are the only thing scoping the result.
+     */
+    public EventsFilterOptions filterOptions(EventsFilterOptionsRequest request, Subject subject) {
+        final var tags = request.fields().contains(EventDto.FIELD_TAGS)
+                ? distinctValues(EventDto.FIELD_TAGS, request, subject)
+                : null;
+
+        return new EventsFilterOptions(tags);
+    }
+
+    /**
+     * Returns the distinct values of the given field, most used first, optionally narrowed to values
+     * containing the request's field query.
+     */
+    private List<String> distinctValues(String field, EventsFilterOptionsRequest request, Subject subject) {
+        final var eventStreams = allowedEventStreams(subject);
+        if (eventStreams.isEmpty()) {
+            return List.of();
+        }
+
+        final var bucketPattern = isNullOrEmpty(request.fieldQuery()) ? null : containsPattern(request.fieldQuery());
+        return moreSearch.aggregateSlicesForColumn(request.query(), request.timerange(), eventStreams, "",
+                        allowedSourceStreams(subject), field, bucketPattern, Map.of(), MAX_FILTER_OPTIONS)
+                .stream()
+                .map(Slice::value)
+                .filter(value -> !isNullOrEmpty(value))
+                .toList();
+    }
+
+    /**
+     * Builds a Lucene regular expression matching values that contain the given text. Lucene regexes
+     * have no case-insensitivity flag, but tags are lowercased at write time (see TagNormalizer), so
+     * lowercasing the input suffices. Revisit before reusing for fields that aren't normalized this
+     * way. Escaping every non-alphanumeric character keeps the input literal.
+     */
+    private static String containsPattern(String fieldQuery) {
+        final var escaped = new StringBuilder();
+        // Iterate code points so surrogate pairs aren't escaped as two broken halves.
+        fieldQuery.toLowerCase(Locale.ROOT).codePoints().forEach(codePoint -> {
+            if (!Character.isLetterOrDigit(codePoint)) {
+                escaped.append('\\');
+            }
+            escaped.appendCodePoint(codePoint);
+        });
+        return ".*" + escaped + ".*";
+    }
+
     private AbsoluteRange effectiveTimeRange(EventsSearchParameters parameters) {
         return AbsoluteRange.create(parameters.timerange().getFrom(), parameters.timerange().getTo());
     }

graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java

@@ -17,6 +17,7 @@
 package org.graylog.events.search;
 
 import com.google.auto.value.AutoValue;
+import jakarta.annotation.Nullable;
 import jakarta.inject.Inject;
 import org.graylog.events.processor.EventProcessorException;
 import org.graylog.plugins.views.search.IndexRangeContainsOneOfStreams;
@@ -203,13 +204,24 @@ private String decorateQuery(String queryString, Set<Parameter> queryParameters)
     public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set<String> eventStreams,
                                        String filterString, SourceStreamFilter sourceStreamFilter,
                                        String slicingColumn, Map<String, Object> meta, int maxBuckets) {
+        return aggregateSlicesForColumn(queryString, timeRange, eventStreams, filterString, sourceStreamFilter,
+                slicingColumn, null, meta, maxBuckets);
+    }
+
+    /**
+     * @param bucketPattern optional Lucene regular expression the returned slice values must match,
+     *                      see {@link MoreSearchAdapter#aggregateSlicesForColumn}
+     */
+    public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set<String> eventStreams,
+                                       String filterString, SourceStreamFilter sourceStreamFilter,
+                                       String slicingColumn, @Nullable String bucketPattern, Map<String, Object> meta, int maxBuckets) {
         final Set<String> affectedIndices = getAffectedIndices(eventStreams, timeRange);
         if (affectedIndices == null || affectedIndices.isEmpty()) {
             return List.of();
         }
         // TODO: add extra filters if necessary
         return moreSearchAdapter.aggregateSlicesForColumn(queryString, timeRange, affectedIndices, eventStreams,
-                filterString, sourceStreamFilter, Map.of(), slicingColumn, meta, maxBuckets);
+                filterString, sourceStreamFilter, Map.of(), slicingColumn, bucketPattern, meta, maxBuckets);
     }
 
     public List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange timeRange, Set<String> eventStreams,

graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.events.search;
 
+import jakarta.annotation.Nullable;
 import org.graylog.events.processor.EventProcessorException;
 import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
 import org.graylog.plugins.views.search.searchtypes.pivot.buckets.NumberRange;
@@ -55,9 +56,15 @@ interface ScrollEventsCallback {
     void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams,
                       List<UsedSearchFilter> filters, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException;
 
+    /**
+     * @param bucketPattern optional Lucene regular expression applied to the bucket keys of the terms
+     *                      aggregation, so only matching values are returned. Pass {@code null} to
+     *                      return all values.
+     */
     List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timerange, Set<String> affectedIndices,
                                 Set<String> eventStreams, String filterString, SourceStreamFilter sourceStreamFilter,
-                                Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, int maxBuckets);
+                                Map<String, Set<String>> extraFilters, String slicingColumn, @Nullable String bucketPattern,
+                                Map<String, Object> meta, int maxBuckets);
 
     List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange timerange, Set<String> affectedIndices,
                                            Set<String> eventStreams, String filterString, SourceStreamFilter sourceStreamFilter,