From d3befd8efc99b07573bf5590f36938d744e8848b Mon Sep 17 00:00:00 2001 From: Dan Torrey Date: Thu, 30 Jul 2026 18:06:53 -0500 Subject: [PATCH 1/2] Add /events/filter_options with source-stream-scoped tag values The tags filter dropdown fetched its values via /events/slices, which goes through the views search engine and requires streams:read on the events stream, returning a 403 for users who can only read the events' source streams. The new endpoint aggregates distinct tag values directly via MoreSearch, scoped to the caller's readable source streams. The dropdown's typed text is forwarded as field_query and applied as a terms-aggregation include pattern (case-insensitive contains), with the response capped at the 50 most-used values. Fixes Graylog2/graylog-plugin-enterprise#14282 Co-Authored-By: Claude Fable 5 --- .../elasticsearch7/MoreSearchAdapterES7.java | 7 +- .../opensearch2/MoreSearchAdapterOS2.java | 10 +- .../opensearch3/MoreSearchAdapterOS.java | 12 +- .../graylog/events/rest/EventsResource.java | 10 + .../events/search/EventsFilterOptions.java | 30 +++ .../search/EventsFilterOptionsRequest.java | 66 ++++++ .../events/search/EventsSearchService.java | 57 +++++ .../org/graylog/events/search/MoreSearch.java | 14 +- .../events/search/MoreSearchAdapter.java | 9 +- .../EventsSearchServiceFilterOptionsTest.java | 213 ++++++++++++++++++ .../MoreSearchAdapterAggregationIT.java | 109 ++++++++- .../more_search_adapter_aggregation.json | 60 +++++ .../src/components/events/TagsFilter.test.tsx | 98 ++++---- .../src/components/events/TagsFilter.tsx | 47 +--- 14 files changed, 650 insertions(+), 92 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java create mode 100644 graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java create mode 100644 graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java index 9aeeb5a1f466..34554787842a 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java @@ -18,6 +18,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -330,10 +331,14 @@ public List aggregateSlices(String queryString, TimeRange timerange, Set< @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { final var builder = AggregationBuilders.terms(slicesAggregationName) .field(slicingColumn) .size(maxBuckets); + if (bucketPattern != null) { + builder.includeExclude(new IncludeExclude(bucketPattern, null)); + } return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java index 94d85396f435..f097d339ca5b 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java @@ -18,6 +18,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -49,6 +50,7 @@ import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.IncludeExclude; import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.builder.SearchSourceBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.FieldSortBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.SortOrder; @@ -332,10 +334,14 @@ private List aggregateSlices(String queryString, TimeRange timerange, Set @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { - AggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName) + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { + TermsAggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName) .field(slicingColumn) .size(maxBuckets); + if (bucketPattern != null) { + builder.includeExclude(new IncludeExclude(bucketPattern, null)); + } return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder); } diff --git a/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java b/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java index d0c636d4f669..d1a794554127 100644 --- a/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java +++ b/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java @@ -17,6 +17,7 @@ package org.graylog.storage.opensearch3; import com.google.common.base.Stopwatch; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -343,10 +344,17 @@ private org.opensearch.client.opensearch._types.SortOrder sortOrder(Sorting sort @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { final var filter = createQuery(queryString, timerange, eventStreams, filterString, sourceStreamFilter, extraFilters); final var aggregation = Aggregation.builder() - .terms(terms -> terms.field(slicingColumn).size(maxBuckets)) + .terms(terms -> { + terms.field(slicingColumn).size(maxBuckets); + if (bucketPattern != null) { + terms.include(inc -> inc.regexp(bucketPattern)); + } + return terms; + }) .build(); final var searchResult = executeAggregation(filter, affectedIndices, SLICES_AGGREGATION_NAME, aggregation); diff --git a/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java b/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java index 4e039ca977ce..cbfe87d93276 100644 --- a/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java +++ b/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java @@ -29,6 +29,8 @@ import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.graylog.events.search.EventsFilterOptions; +import org.graylog.events.search.EventsFilterOptionsRequest; import org.graylog.events.search.EventsHistogramResult; import org.graylog.events.search.EventsSearchParameters; import org.graylog.events.search.EventsSearchResult; @@ -88,6 +90,14 @@ public Slices slices(@Context SearchUser searchUser, @Parameter(name = "JSON bod return sliceService.slices(firstNonNull(request, EventsSlicesRequest.empty()), getSubject(), searchUser); } + @POST + @Path("/filter_options") + @Operation(summary = "Get the available values for the given event fields") + @NoAuditEvent("Doesn't change any data, only collects filter values") + public EventsFilterOptions filterOptions(@Parameter(name = "JSON body") final EventsFilterOptionsRequest request) { + return searchService.filterOptions(firstNonNull(request, EventsFilterOptionsRequest.empty()), getSubject()); + } + @POST @Path("/histogram") @Operation(summary = "Build histogram of events over time") diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java new file mode 100644 index 000000000000..ec8c71014ad9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.events.event.EventDto; + +import java.util.List; + +/** + * Values available to filter the events table by. Fields that were not requested are omitted. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record EventsFilterOptions(@JsonProperty(EventDto.FIELD_TAGS) List tags) { +} diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java new file mode 100644 index 000000000000..d883f05690ea --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; +import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; + +import java.util.List; + +/** + * Request for the values available to filter the events table by. + * + * @param fields the event fields to return available values for + * @param query optional query to narrow the events the values are collected from, e.g. to scope + * the result to a single source stream + * @param fieldQuery optional search text the returned values must contain (case-insensitive) + * @param timerange the time range to collect values from, defaulting to the last 30 days + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record EventsFilterOptionsRequest(@JsonProperty("fields") List fields, + @JsonProperty("query") String query, + @JsonProperty("field_query") String fieldQuery, + @JsonProperty("timerange") TimeRange timerange) { + + private static final int DEFAULT_RANGE_SECONDS = 30 * 24 * 60 * 60; + + // Defensive cap: field values are short (tags max out at 128 chars), so longer search text can + // never match and would only bloat the pattern sent to the search backend. + private static final int MAX_FIELD_QUERY_LENGTH = 256; + + public EventsFilterOptionsRequest { + fields = fields == null ? List.of() : fields; + query = query == null ? "" : query; + fieldQuery = fieldQuery == null ? "" : fieldQuery.substring(0, Math.min(fieldQuery.length(), MAX_FIELD_QUERY_LENGTH)); + timerange = timerange == null ? defaultTimerange() : timerange; + } + + public static EventsFilterOptionsRequest empty() { + return new EventsFilterOptionsRequest(List.of(), "", "", null); + } + + private static TimeRange defaultTimerange() { + try { + return RelativeRange.create(DEFAULT_RANGE_SECONDS); + } catch (InvalidRangeParametersException e) { + throw new IllegalStateException("Unable to create the default filter options time range", e); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java index 800c7a958ec6..b6732b66dde5 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java @@ -25,17 +25,24 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.rest.resources.entities.Slice; import org.graylog2.streams.StreamService; import java.time.ZoneId; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import static com.google.common.base.Strings.isNullOrEmpty; import static org.graylog.events.search.EventsSearchFilter.NULL_VALUE; public class EventsSearchService extends AbstractEventsSearchService { + // Upper bound on the values returned per filter option field. The dropdown shows the most-used + // values first and relies on the field query for anything beyond the cap. + private static final int MAX_FILTER_OPTIONS = 50; + private final MoreSearch moreSearch; private final StreamService streamService; @@ -95,6 +102,56 @@ public EventsHistogramResult histogram(EventsSearchParameters parameters, Subjec return EventsHistogramResult.fromResult(result); } + /** + * Returns the values the events table can be filtered by, collected from the events the subject is + * permitted to see. Deliberately aggregates through {@link MoreSearch} rather than the views search + * engine so that the source stream permissions of the subject are the only thing scoping the result. + */ + public EventsFilterOptions filterOptions(EventsFilterOptionsRequest request, Subject subject) { + final var tags = request.fields().contains(EventDto.FIELD_TAGS) + ? distinctValues(EventDto.FIELD_TAGS, request, subject) + : null; + + return new EventsFilterOptions(tags); + } + + /** + * Returns the distinct values of the given field, most used first, optionally narrowed to values + * containing the request's field query. + */ + private List distinctValues(String field, EventsFilterOptionsRequest request, Subject subject) { + final var eventStreams = allowedEventStreams(subject); + if (eventStreams.isEmpty()) { + return List.of(); + } + + final var bucketPattern = isNullOrEmpty(request.fieldQuery()) ? null : containsPattern(request.fieldQuery()); + return moreSearch.aggregateSlicesForColumn(request.query(), request.timerange(), eventStreams, "", + allowedSourceStreams(subject), field, bucketPattern, Map.of(), MAX_FILTER_OPTIONS) + .stream() + .map(Slice::value) + .filter(value -> !isNullOrEmpty(value)) + .toList(); + } + + /** + * Builds a Lucene regular expression matching values that contain the given text. Lucene regexes + * have no case-insensitivity flag, but tags are lowercased at write time (see TagNormalizer), so + * lowercasing the input suffices. Revisit before reusing for fields that aren't normalized this + * way. Escaping every non-alphanumeric character keeps the input literal. + */ + private static String containsPattern(String fieldQuery) { + final var escaped = new StringBuilder(); + // Iterate code points so surrogate pairs aren't escaped as two broken halves. + fieldQuery.toLowerCase(Locale.ROOT).codePoints().forEach(codePoint -> { + if (!Character.isLetterOrDigit(codePoint)) { + escaped.append('\\'); + } + escaped.appendCodePoint(codePoint); + }); + return ".*" + escaped + ".*"; + } + private AbsoluteRange effectiveTimeRange(EventsSearchParameters parameters) { return AbsoluteRange.create(parameters.timerange().getFrom(), parameters.timerange().getTo()); } diff --git a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java index 15c914f1799e..9196ea8eff78 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java @@ -17,6 +17,7 @@ package org.graylog.events.search; import com.google.auto.value.AutoValue; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import org.graylog.events.processor.EventProcessorException; import org.graylog.plugins.views.search.IndexRangeContainsOneOfStreams; @@ -203,13 +204,24 @@ private String decorateQuery(String queryString, Set queryParameters) public List aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, String slicingColumn, Map meta, int maxBuckets) { + return aggregateSlicesForColumn(queryString, timeRange, eventStreams, filterString, sourceStreamFilter, + slicingColumn, null, meta, maxBuckets); + } + + /** + * @param bucketPattern optional Lucene regular expression the returned slice values must match, + * see {@link MoreSearchAdapter#aggregateSlicesForColumn} + */ + public List aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set eventStreams, + String filterString, SourceStreamFilter sourceStreamFilter, + String slicingColumn, @Nullable String bucketPattern, Map meta, int maxBuckets) { final Set affectedIndices = getAffectedIndices(eventStreams, timeRange); if (affectedIndices == null || affectedIndices.isEmpty()) { return List.of(); } // TODO: add extra filters if necessary return moreSearchAdapter.aggregateSlicesForColumn(queryString, timeRange, affectedIndices, eventStreams, - filterString, sourceStreamFilter, Map.of(), slicingColumn, meta, maxBuckets); + filterString, sourceStreamFilter, Map.of(), slicingColumn, bucketPattern, meta, maxBuckets); } public List aggregateSlicesForRangeQuery(String queryString, TimeRange timeRange, Set eventStreams, diff --git a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java index aa70920a6216..abecf578c3ba 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java @@ -16,6 +16,7 @@ */ package org.graylog.events.search; +import jakarta.annotation.Nullable; import org.graylog.events.processor.EventProcessorException; import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter; import org.graylog.plugins.views.search.searchtypes.pivot.buckets.NumberRange; @@ -55,9 +56,15 @@ interface ScrollEventsCallback { void scrollEvents(String queryString, TimeRange timeRange, Set affectedIndices, Set streams, List filters, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException; + /** + * @param bucketPattern optional Lucene regular expression applied to the bucket keys of the terms + * aggregation, so only matching values are returned. Pass {@code null} to + * return all values. + */ List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets); + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets); List aggregateSlicesForRangeQuery(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, diff --git a/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java b/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java new file mode 100644 index 000000000000..1234792aead2 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.shiro.subject.Subject; +import org.graylog.events.event.EventDto; +import org.graylog.events.processor.DBEventDefinitionService; +import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.rest.resources.entities.Slice; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.streams.StreamService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.graylog2.plugin.streams.Stream.DEFAULT_EVENTS_STREAM_ID; +import static org.graylog2.plugin.streams.Stream.DEFAULT_SYSTEM_EVENTS_STREAM_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EventsSearchServiceFilterOptionsTest { + private static final String SOURCE_STREAM = "source-stream-allowed"; + private static final String OTHER_STREAM = "source-stream-denied"; + + @Mock + private MoreSearch moreSearch; + @Mock + private StreamService streamService; + @Mock + private DBEventDefinitionService eventDefinitionService; + @Mock + private ObjectMapper objectMapper; + @Mock + private Subject subject; + + private EventsSearchService service; + + @BeforeEach + void setUp() { + service = new EventsSearchService(moreSearch, streamService, eventDefinitionService, objectMapper); + } + + @Test + void returnsTagsInAggregationOrderForSubjectWithoutGlobalStreamPermission() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"), slice("credential-access"), slice("linux"))); + + final var result = service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + // The aggregation returns the most used values first; that order must be preserved. + assertThat(result.tags()).containsExactly("windows", "credential-access", "linux"); + } + + @Test + void scopesAggregationToTheSourceStreamsTheSubjectMayRead() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + final ArgumentCaptor> eventStreams = ArgumentCaptor.forClass(Set.class); + final ArgumentCaptor sourceStreams = ArgumentCaptor.forClass(SourceStreamFilter.class); + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), eventStreams.capture(), anyString(), + sourceStreams.capture(), eq(EventDto.FIELD_TAGS), any(), anyMap(), eq(50)); + + assertThat(eventStreams.getValue()).containsExactly(DEFAULT_EVENTS_STREAM_ID); + assertThat(sourceStreams.getValue().isAllAllowed()).isFalse(); + assertThat(sourceStreams.getValue().streamIds()).containsExactly(SOURCE_STREAM); + } + + @Test + void passesTheRequestQueryToTheAggregation() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "source_streams:" + SOURCE_STREAM), subject); + + verify(moreSearch).aggregateSlicesForColumn(eq("source_streams:" + SOURCE_STREAM), any(), anySet(), + anyString(), any(), eq(EventDto.FIELD_TAGS), isNull(), anyMap(), anyInt()); + } + + @Test + void buildsAContainsPatternFromTheFieldQuery() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("credential-access"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "Access"), subject); + + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*access.*"), anyMap(), anyInt()); + } + + @Test + void escapesSupplementaryCharactersInTheFieldQueryAsWholeCodePoints() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of()); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "a🦊b"), subject); + + // The surrogate pair must stay together behind a single escape, not become two escaped halves. + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*a\\🦊b.*"), anyMap(), anyInt()); + } + + @Test + void capsTheFieldQueryLength() { + final var request = new EventsFilterOptionsRequest(List.of(), "", "a".repeat(1000), null); + + assertThat(request.fieldQuery()).hasSize(256); + } + + @Test + void appliesRequestDefaults() throws Exception { + final var request = new EventsFilterOptionsRequest(null, null, null, null); + + assertThat(request.fields()).isEmpty(); + assertThat(request.query()).isEmpty(); + assertThat(request.fieldQuery()).isEmpty(); + // The default time range for collecting filter values is the last 30 days. + assertThat(request.timerange()).isEqualTo(RelativeRange.create(30 * 24 * 60 * 60)); + } + + @Test + void escapesRegexMetacharactersInTheFieldQuery() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("t1003.001"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "T1003.001"), subject); + + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*t1003\\.001.*"), anyMap(), anyInt()); + } + + @Test + void skipsValuesWithoutATag() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"), slice(null), slice(""))); + + final var result = service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + assertThat(result.tags()).containsExactly("windows"); + } + + @Test + void omitsFieldsThatWereNotRequested() { + final var result = service.filterOptions(request(List.of(), ""), subject); + + assertThat(result.tags()).isNull(); + verifyNoInteractions(moreSearch); + } + + private void mockSourceStreamOnlyPermissions() { + when(subject.isPermitted(RestPermissions.STREAMS_READ)).thenReturn(false); + when(subject.isPermitted(permission(DEFAULT_EVENTS_STREAM_ID))).thenReturn(false); + when(subject.isPermitted(permission(DEFAULT_SYSTEM_EVENTS_STREAM_ID))).thenReturn(false); + when(subject.isPermitted(permission(SOURCE_STREAM))).thenReturn(true); + when(subject.isPermitted(permission(OTHER_STREAM))).thenReturn(false); + when(streamService.streamAllIds()).thenAnswer(invocation -> java.util.stream.Stream.of(SOURCE_STREAM, OTHER_STREAM)); + } + + private void mockAggregation(List slices) { + when(moreSearch.aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), anyString(), + any(), anyMap(), anyInt())).thenReturn(slices); + } + + private static EventsFilterOptionsRequest request(List fields, String query) { + return request(fields, query, ""); + } + + private static EventsFilterOptionsRequest request(List fields, String query, String fieldQuery) { + return new EventsFilterOptionsRequest(fields, query, fieldQuery, RelativeRange.allTime()); + } + + private static String permission(String streamId) { + return String.join(":", RestPermissions.STREAMS_READ, streamId); + } + + private static Slice slice(String value) { + return new Slice(value, null, 1, Map.of()); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java b/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java index effa3ef132fe..f15e544029a7 100644 --- a/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java +++ b/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java @@ -34,6 +34,7 @@ public abstract class MoreSearchAdapterAggregationIT extends ElasticsearchBaseTe private static final String INDEX_NAME = "graylog_0"; private static final Set ALL_STREAMS = Set.of("stream-a", "stream-b"); + private static final String EVENTS_STREAM = "000000000000000000000002"; private MoreSearchAdapter adapter; @@ -52,7 +53,7 @@ public void aggregateSlicesForColumn_groupsByField() { final List result = adapter.aggregateSlicesForColumn( "*", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "gl2_source_input", Map.of(), 100); + Map.of(), "gl2_source_input", null, Map.of(), 100); final Map countsByInput = result.stream() .collect(Collectors.toMap(Slice::value, Slice::count)); @@ -68,7 +69,7 @@ public void aggregateSlicesForColumn_withQueryFilter() { final List result = adapter.aggregateSlicesForColumn( "gl2_source_input:input-1", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "streams", Map.of(), 100); + Map.of(), "streams", null, Map.of(), 100); final Map countsByStream = result.stream() .collect(Collectors.toMap(Slice::value, Slice::count)); @@ -83,11 +84,113 @@ public void aggregateSlicesForColumn_emptyResultForNoMatch() { final List result = adapter.aggregateSlicesForColumn( "gl2_source_input:nonexistent", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "streams", Map.of(), 100); + Map.of(), "streams", null, Map.of(), 100); assertThat(result).isEmpty(); } + // --- aggregateSlicesForColumn source stream permission tests --- + // + // The events table and its filter dropdowns rely on the SourceStreamFilter argument alone to keep a + // user from seeing values off events they may not read, so assert it actually filters rather than + // trusting the caller wiring. + + @Test + public void aggregateSlicesForColumn_onlyReturnsValuesFromAllowedSourceStreams() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-a")), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("windows", 2, "execution", 1)); + } + + @Test + public void aggregateSlicesForColumn_returnsValuesForADifferentAllowedSourceStream() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-b")), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("linux", 1, "credential-access", 1)); + } + + @Test + public void aggregateSlicesForColumn_returnsValuesFromEverySourceStreamWhenAllAllowed() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("windows", "execution", "linux", "credential-access", + "t1003.001", "t1003x001", "access-token"); + } + + @Test + public void aggregateSlicesForColumn_returnsNothingForAnEmptySourceStreamAllowList() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of()), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(result).isEmpty(); + } + + // --- aggregateSlicesForColumn bucket pattern tests --- + // + // The bucket pattern backs the server-side search of the tags filter dropdown. It must filter the + // aggregated values themselves (not the events), stay literal for regex metacharacters, and compose + // with the source stream allow-list. + + @Test + public void aggregateSlicesForColumn_bucketPatternOnlyReturnsMatchingValues() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*access.*", Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("credential-access", 1, "access-token", 1)); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternTreatsEscapedMetacharactersLiterally() { + // An unescaped "." would also match the "t1003x001" tag. + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*t1003\\.001.*", Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("t1003.001"); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternComposesWithSourceStreamAllowList() { + // "access" tags exist in stream-b and stream-c; only the stream-b one may be returned. + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-b")), + Map.of(), "tags", ".*access.*", Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("credential-access"); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternWithoutMatchesReturnsNothing() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*doesnotexist.*", Map.of(), 100); + + assertThat(result).isEmpty(); + } + + private static Map tagCounts(List slices) { + return slices.stream().collect(Collectors.toMap(Slice::value, Slice::count)); + } + // --- aggregateGroupedTerms tests --- @Test diff --git a/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json b/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json index 82e45bc398c4..51ff47502814 100644 --- a/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json +++ b/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json @@ -74,6 +74,66 @@ } } ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-0" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-a", + "timestamp": "2024-01-01 06:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-a"], + "tags": ["windows", "execution"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-1" } }, + { + "data": { + "source": "localhost", + "message": "second event in stream-a", + "timestamp": "2024-01-01 07:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-a"], + "tags": ["windows"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-2" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-b", + "timestamp": "2024-01-01 08:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-b"], + "tags": ["linux", "credential-access"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-3" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-c", + "timestamp": "2024-01-01 09:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-c"], + "tags": ["t1003.001", "t1003x001", "access-token"] + } + } + ] } ] } diff --git a/graylog2-web-interface/src/components/events/TagsFilter.test.tsx b/graylog2-web-interface/src/components/events/TagsFilter.test.tsx index 49dee3b90e27..53374ffd7fca 100644 --- a/graylog2-web-interface/src/components/events/TagsFilter.test.tsx +++ b/graylog2-web-interface/src/components/events/TagsFilter.test.tsx @@ -16,13 +16,14 @@ */ import * as React from 'react'; import { render, screen, waitFor } from 'wrappedTestingLibrary'; +import userEvent from '@testing-library/user-event'; import { Events } from '@graylog/server-api'; import TagsFilter from 'components/events/TagsFilter'; jest.mock('@graylog/server-api', () => ({ - Events: { slices: jest.fn() }, + Events: { filterOptions: jest.fn() }, })); jest.mock('routing/useQuery', () => ({ @@ -30,7 +31,10 @@ jest.mock('routing/useQuery', () => ({ default: () => ({}), })); -const mockedSlices = Events.slices as jest.MockedFunction; +const mockedFilterOptions = Events.filterOptions as jest.MockedFunction; + +const filterOptionsResponse = (tags: Array) => + ({ tags }) as unknown as Awaited>; const tagsAttribute = { id: 'tags', @@ -39,72 +43,74 @@ const tagsAttribute = { filterable: true, } as const; +const renderFilter = () => + render( + t} + onSubmit={jest.fn()} + />, + ); + describe('TagsFilter', () => { beforeEach(() => { - mockedSlices.mockReset(); + mockedFilterOptions.mockReset(); }); - it('queries Events.slices with slice_column=tags and renders returned values', async () => { - mockedSlices.mockResolvedValue({ - slices: [ - { value: 'phishing', count: 5, title: null, meta: {} }, - { value: 'exfil', count: 2, title: null, meta: {} }, - ], - } as unknown as Awaited>); - - render( - t} - onSubmit={jest.fn()} - />, - ); + it('requests the tags filter options and renders returned values', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse(['exfil', 'phishing'])); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalledWith(expect.objectContaining({ slice_column: 'tags', include_all: true })); + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ fields: ['tags'] })); }); expect(await screen.findByText('exfil')).toBeInTheDocument(); expect(await screen.findByText('phishing')).toBeInTheDocument(); }); - it('renders an empty list when slices returns nothing', async () => { - mockedSlices.mockResolvedValue({ slices: [] } as unknown as Awaited>); - - render( - t} - onSubmit={jest.fn()} - />, - ); + it('renders an empty list when no tags are returned', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse([])); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalled(); + expect(mockedFilterOptions).toHaveBeenCalled(); }); expect(screen.queryByText('phishing')).not.toBeInTheDocument(); }); - it('falls back to empty suggestions when the request fails', async () => { - mockedSlices.mockRejectedValue(new Error('boom')); - - render( - t} - onSubmit={jest.fn()} - />, + it('forwards the typed search query to the server', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse(['credential-access'])); + + renderFilter(); + + await waitFor(() => { + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ field_query: '' })); + }); + + await userEvent.type(await screen.findByPlaceholderText('Search for tags'), 'access'); + + // The search box debounces for 1s before the query is forwarded. + await waitFor( + () => { + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ field_query: 'access' })); + }, + { timeout: 5000 }, ); + }, 10000); + + it('falls back to empty suggestions when the request fails', async () => { + mockedFilterOptions.mockRejectedValue(new Error('boom')); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalled(); + expect(mockedFilterOptions).toHaveBeenCalled(); }); expect(screen.queryByText('phishing')).not.toBeInTheDocument(); diff --git a/graylog2-web-interface/src/components/events/TagsFilter.tsx b/graylog2-web-interface/src/components/events/TagsFilter.tsx index 2a3ff9eea41a..5475ee5d7a91 100644 --- a/graylog2-web-interface/src/components/events/TagsFilter.tsx +++ b/graylog2-web-interface/src/components/events/TagsFilter.tsx @@ -23,8 +23,6 @@ import { Events } from '@graylog/server-api'; import type { FilterComponentProps } from 'stores/PaginationTypes'; import SuggestionsList from 'components/common/EntityFilters/FilterConfiguration/SuggestionsList'; import useQuery_ from 'routing/useQuery'; -import useDebouncedValue from 'hooks/useDebouncedValue'; -import { MISSING_BUCKET_NAME } from 'views/Constants'; const DEFAULT_SEARCH_PARAMS = { query: '', @@ -39,39 +37,15 @@ const TIMERANGE_30D = { type: 'relative', range: 30 * 24 * 60 * 60 }; // Hard cap on rendered suggestions; relies on the search box to narrow further. const MAX_RENDERED_SUGGESTIONS = 100; -// Debounce window for forwarding the typed prefix to a server-side fetcher. -const PREFIX_DEBOUNCE_MS = 300; - type Suggestion = { id: string; value: string }; -const emptyFilter = { - alerts: 'include' as const, - extra_filters: {}, - aggregation_timerange: { type: 'relative', range: 0 }, - id: [] as string[], - priority: [] as string[], - event_definitions: [] as string[], - key: [] as string[], -}; - -const fetchTagSuggestions = (streamId: string | undefined, _prefix?: string): Promise => - Events.slices({ - include_all: true, - slice_column: 'tags', +const fetchTagSuggestions = (streamId: string | undefined, prefix?: string): Promise => + Events.filterOptions({ + fields: ['tags'], query: streamId ? `source_streams:${streamId}` : '', - filter: emptyFilter, + field_query: prefix ?? '', timerange: TIMERANGE_30D, - }).then((response) => - (response?.slices ?? []) - .map((slice) => slice?.value) - // Drop the scripting-API missing-bucket placeholder — events without tags would otherwise - // surface as a literal "(Empty Value)" suggestion, which isn't a real tag to filter by. - .filter( - (value): value is string => typeof value === 'string' && value.length > 0 && value !== MISSING_BUCKET_NAME, - ) - .sort() - .map((value) => ({ id: value, value })), - ); + }).then((response) => (response?.tags ?? []).map((value) => ({ id: value, value }))); type FetchSuggestions = (streamId: string | undefined, prefix?: string) => Promise; @@ -80,9 +54,10 @@ type FetchSuggestions = (streamId: string | undefined, prefix?: string) => Promi * truth (indexed events vs. defined event definitions) so the fetcher is injected. * * Set `serverSidePrefix` when the backend caps results (e.g. the event-definitions - * suggest endpoint, which is hard-capped at 100). In that mode the debounced search-box - * query is forwarded to the fetcher as a prefix and included in the React Query key so - * later keystrokes can reach entries past the server cap. + * suggest endpoint, which is hard-capped at 100). In that mode the search-box query + * (already debounced by the suggestions list) is forwarded to the fetcher as a prefix + * and included in the React Query key so later keystrokes can reach entries past the + * server cap. */ export const createTagsFilter = ({ queryKeyPrefix, @@ -97,10 +72,9 @@ export const createTagsFilter = ({ }) => { const Filter = ({ attribute, allActiveFilters, filter, filterValueRenderer, onSubmit }: FilterComponentProps) => { const [searchParams, setSearchParams] = useState(DEFAULT_SEARCH_PARAMS); - const [debouncedQuery] = useDebouncedValue(searchParams.query, PREFIX_DEBOUNCE_MS); const { stream_id: streamId } = useQuery_(); const streamIdParam = streamScoped && typeof streamId === 'string' ? streamId : undefined; - const prefixParam = serverSidePrefix ? debouncedQuery : undefined; + const prefixParam = serverSidePrefix ? searchParams.query : undefined; const { data: allSuggestions, isInitialLoading, @@ -149,6 +123,7 @@ const TagsFilter = createTagsFilter({ queryKeyPrefix: ['events', 'tag-suggestions', `t${TIMERANGE_30D.range}`], fetchSuggestions: fetchTagSuggestions, streamScoped: true, + serverSidePrefix: true, }); export default TagsFilter; From 09f93cef7dfff5f9ccba98379fe9b6d8e63692b5 Mon Sep 17 00:00:00 2001 From: Dan Torrey Date: Thu, 30 Jul 2026 18:23:39 -0500 Subject: [PATCH 2/2] Add PR to the event definition tags changelog entry Co-Authored-By: Claude Fable 5 --- changelog/unreleased/pr-25896.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/pr-25896.toml b/changelog/unreleased/pr-25896.toml index d1307eae4948..45ccc17ed5d2 100644 --- a/changelog/unreleased/pr-25896.toml +++ b/changelog/unreleased/pr-25896.toml @@ -1,4 +1,4 @@ type = "a" message = "Ability to assign tags to event definitions." -pulls = ["25896", "25940", "26079"] +pulls = ["25896", "25940", "26079", "26841"]