diff --git a/changelog/unreleased/pr-25896.toml b/changelog/unreleased/pr-25896.toml index d1307eae4948..45ccc17ed5d2 100644 --- a/changelog/unreleased/pr-25896.toml +++ b/changelog/unreleased/pr-25896.toml @@ -1,4 +1,4 @@ type = "a" message = "Ability to assign tags to event definitions." -pulls = ["25896", "25940", "26079"] +pulls = ["25896", "25940", "26079", "26841"] diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java index 9aeeb5a1f466..34554787842a 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java @@ -18,6 +18,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -330,10 +331,14 @@ public List aggregateSlices(String queryString, TimeRange timerange, Set< @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { final var builder = AggregationBuilders.terms(slicesAggregationName) .field(slicingColumn) .size(maxBuckets); + if (bucketPattern != null) { + builder.includeExclude(new IncludeExclude(bucketPattern, null)); + } return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java index 94d85396f435..f097d339ca5b 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java @@ -18,6 +18,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -49,6 +50,7 @@ import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.IncludeExclude; import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.builder.SearchSourceBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.FieldSortBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.SortOrder; @@ -332,10 +334,14 @@ private List aggregateSlices(String queryString, TimeRange timerange, Set @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { - AggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName) + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { + TermsAggregationBuilder builder = AggregationBuilders.terms(slicesAggregationName) .field(slicingColumn) .size(maxBuckets); + if (bucketPattern != null) { + builder.includeExclude(new IncludeExclude(bucketPattern, null)); + } return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, sourceStreamFilter, extraFilters, meta, builder); } diff --git a/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java b/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java index d0c636d4f669..d1a794554127 100644 --- a/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java +++ b/graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/MoreSearchAdapterOS.java @@ -17,6 +17,7 @@ package org.graylog.storage.opensearch3; import com.google.common.base.Stopwatch; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; import org.graylog.events.event.EventDto; @@ -343,10 +344,17 @@ private org.opensearch.client.opensearch._types.SortOrder sortOrder(Sorting sort @Override public List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets) { + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets) { final var filter = createQuery(queryString, timerange, eventStreams, filterString, sourceStreamFilter, extraFilters); final var aggregation = Aggregation.builder() - .terms(terms -> terms.field(slicingColumn).size(maxBuckets)) + .terms(terms -> { + terms.field(slicingColumn).size(maxBuckets); + if (bucketPattern != null) { + terms.include(inc -> inc.regexp(bucketPattern)); + } + return terms; + }) .build(); final var searchResult = executeAggregation(filter, affectedIndices, SLICES_AGGREGATION_NAME, aggregation); diff --git a/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java b/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java index 4e039ca977ce..cbfe87d93276 100644 --- a/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java +++ b/graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java @@ -29,6 +29,8 @@ import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.graylog.events.search.EventsFilterOptions; +import org.graylog.events.search.EventsFilterOptionsRequest; import org.graylog.events.search.EventsHistogramResult; import org.graylog.events.search.EventsSearchParameters; import org.graylog.events.search.EventsSearchResult; @@ -88,6 +90,14 @@ public Slices slices(@Context SearchUser searchUser, @Parameter(name = "JSON bod return sliceService.slices(firstNonNull(request, EventsSlicesRequest.empty()), getSubject(), searchUser); } + @POST + @Path("/filter_options") + @Operation(summary = "Get the available values for the given event fields") + @NoAuditEvent("Doesn't change any data, only collects filter values") + public EventsFilterOptions filterOptions(@Parameter(name = "JSON body") final EventsFilterOptionsRequest request) { + return searchService.filterOptions(firstNonNull(request, EventsFilterOptionsRequest.empty()), getSubject()); + } + @POST @Path("/histogram") @Operation(summary = "Build histogram of events over time") diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java new file mode 100644 index 000000000000..ec8c71014ad9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptions.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.events.event.EventDto; + +import java.util.List; + +/** + * Values available to filter the events table by. Fields that were not requested are omitted. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record EventsFilterOptions(@JsonProperty(EventDto.FIELD_TAGS) List tags) { +} diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java new file mode 100644 index 000000000000..d883f05690ea --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsFilterOptionsRequest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; +import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; + +import java.util.List; + +/** + * Request for the values available to filter the events table by. + * + * @param fields the event fields to return available values for + * @param query optional query to narrow the events the values are collected from, e.g. to scope + * the result to a single source stream + * @param fieldQuery optional search text the returned values must contain (case-insensitive) + * @param timerange the time range to collect values from, defaulting to the last 30 days + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record EventsFilterOptionsRequest(@JsonProperty("fields") List fields, + @JsonProperty("query") String query, + @JsonProperty("field_query") String fieldQuery, + @JsonProperty("timerange") TimeRange timerange) { + + private static final int DEFAULT_RANGE_SECONDS = 30 * 24 * 60 * 60; + + // Defensive cap: field values are short (tags max out at 128 chars), so longer search text can + // never match and would only bloat the pattern sent to the search backend. + private static final int MAX_FIELD_QUERY_LENGTH = 256; + + public EventsFilterOptionsRequest { + fields = fields == null ? List.of() : fields; + query = query == null ? "" : query; + fieldQuery = fieldQuery == null ? "" : fieldQuery.substring(0, Math.min(fieldQuery.length(), MAX_FIELD_QUERY_LENGTH)); + timerange = timerange == null ? defaultTimerange() : timerange; + } + + public static EventsFilterOptionsRequest empty() { + return new EventsFilterOptionsRequest(List.of(), "", "", null); + } + + private static TimeRange defaultTimerange() { + try { + return RelativeRange.create(DEFAULT_RANGE_SECONDS); + } catch (InvalidRangeParametersException e) { + throw new IllegalStateException("Unable to create the default filter options time range", e); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java index 800c7a958ec6..b6732b66dde5 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java @@ -25,17 +25,24 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.rest.resources.entities.Slice; import org.graylog2.streams.StreamService; import java.time.ZoneId; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import static com.google.common.base.Strings.isNullOrEmpty; import static org.graylog.events.search.EventsSearchFilter.NULL_VALUE; public class EventsSearchService extends AbstractEventsSearchService { + // Upper bound on the values returned per filter option field. The dropdown shows the most-used + // values first and relies on the field query for anything beyond the cap. + private static final int MAX_FILTER_OPTIONS = 50; + private final MoreSearch moreSearch; private final StreamService streamService; @@ -95,6 +102,56 @@ public EventsHistogramResult histogram(EventsSearchParameters parameters, Subjec return EventsHistogramResult.fromResult(result); } + /** + * Returns the values the events table can be filtered by, collected from the events the subject is + * permitted to see. Deliberately aggregates through {@link MoreSearch} rather than the views search + * engine so that the source stream permissions of the subject are the only thing scoping the result. + */ + public EventsFilterOptions filterOptions(EventsFilterOptionsRequest request, Subject subject) { + final var tags = request.fields().contains(EventDto.FIELD_TAGS) + ? distinctValues(EventDto.FIELD_TAGS, request, subject) + : null; + + return new EventsFilterOptions(tags); + } + + /** + * Returns the distinct values of the given field, most used first, optionally narrowed to values + * containing the request's field query. + */ + private List distinctValues(String field, EventsFilterOptionsRequest request, Subject subject) { + final var eventStreams = allowedEventStreams(subject); + if (eventStreams.isEmpty()) { + return List.of(); + } + + final var bucketPattern = isNullOrEmpty(request.fieldQuery()) ? null : containsPattern(request.fieldQuery()); + return moreSearch.aggregateSlicesForColumn(request.query(), request.timerange(), eventStreams, "", + allowedSourceStreams(subject), field, bucketPattern, Map.of(), MAX_FILTER_OPTIONS) + .stream() + .map(Slice::value) + .filter(value -> !isNullOrEmpty(value)) + .toList(); + } + + /** + * Builds a Lucene regular expression matching values that contain the given text. Lucene regexes + * have no case-insensitivity flag, but tags are lowercased at write time (see TagNormalizer), so + * lowercasing the input suffices. Revisit before reusing for fields that aren't normalized this + * way. Escaping every non-alphanumeric character keeps the input literal. + */ + private static String containsPattern(String fieldQuery) { + final var escaped = new StringBuilder(); + // Iterate code points so surrogate pairs aren't escaped as two broken halves. + fieldQuery.toLowerCase(Locale.ROOT).codePoints().forEach(codePoint -> { + if (!Character.isLetterOrDigit(codePoint)) { + escaped.append('\\'); + } + escaped.appendCodePoint(codePoint); + }); + return ".*" + escaped + ".*"; + } + private AbsoluteRange effectiveTimeRange(EventsSearchParameters parameters) { return AbsoluteRange.create(parameters.timerange().getFrom(), parameters.timerange().getTo()); } diff --git a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java index 15c914f1799e..9196ea8eff78 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearch.java @@ -17,6 +17,7 @@ package org.graylog.events.search; import com.google.auto.value.AutoValue; +import jakarta.annotation.Nullable; import jakarta.inject.Inject; import org.graylog.events.processor.EventProcessorException; import org.graylog.plugins.views.search.IndexRangeContainsOneOfStreams; @@ -203,13 +204,24 @@ private String decorateQuery(String queryString, Set queryParameters) public List aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, String slicingColumn, Map meta, int maxBuckets) { + return aggregateSlicesForColumn(queryString, timeRange, eventStreams, filterString, sourceStreamFilter, + slicingColumn, null, meta, maxBuckets); + } + + /** + * @param bucketPattern optional Lucene regular expression the returned slice values must match, + * see {@link MoreSearchAdapter#aggregateSlicesForColumn} + */ + public List aggregateSlicesForColumn(String queryString, TimeRange timeRange, Set eventStreams, + String filterString, SourceStreamFilter sourceStreamFilter, + String slicingColumn, @Nullable String bucketPattern, Map meta, int maxBuckets) { final Set affectedIndices = getAffectedIndices(eventStreams, timeRange); if (affectedIndices == null || affectedIndices.isEmpty()) { return List.of(); } // TODO: add extra filters if necessary return moreSearchAdapter.aggregateSlicesForColumn(queryString, timeRange, affectedIndices, eventStreams, - filterString, sourceStreamFilter, Map.of(), slicingColumn, meta, maxBuckets); + filterString, sourceStreamFilter, Map.of(), slicingColumn, bucketPattern, meta, maxBuckets); } public List aggregateSlicesForRangeQuery(String queryString, TimeRange timeRange, Set eventStreams, diff --git a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java index aa70920a6216..abecf578c3ba 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/MoreSearchAdapter.java @@ -16,6 +16,7 @@ */ package org.graylog.events.search; +import jakarta.annotation.Nullable; import org.graylog.events.processor.EventProcessorException; import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter; import org.graylog.plugins.views.search.searchtypes.pivot.buckets.NumberRange; @@ -55,9 +56,15 @@ interface ScrollEventsCallback { void scrollEvents(String queryString, TimeRange timeRange, Set affectedIndices, Set streams, List filters, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException; + /** + * @param bucketPattern optional Lucene regular expression applied to the bucket keys of the terms + * aggregation, so only matching values are returned. Pass {@code null} to + * return all values. + */ List aggregateSlicesForColumn(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, - Map> extraFilters, String slicingColumn, Map meta, int maxBuckets); + Map> extraFilters, String slicingColumn, @Nullable String bucketPattern, + Map meta, int maxBuckets); List aggregateSlicesForRangeQuery(String queryString, TimeRange timerange, Set affectedIndices, Set eventStreams, String filterString, SourceStreamFilter sourceStreamFilter, diff --git a/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java b/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java new file mode 100644 index 000000000000..1234792aead2 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/events/search/EventsSearchServiceFilterOptionsTest.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.events.search; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.shiro.subject.Subject; +import org.graylog.events.event.EventDto; +import org.graylog.events.processor.DBEventDefinitionService; +import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.rest.resources.entities.Slice; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.streams.StreamService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.graylog2.plugin.streams.Stream.DEFAULT_EVENTS_STREAM_ID; +import static org.graylog2.plugin.streams.Stream.DEFAULT_SYSTEM_EVENTS_STREAM_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EventsSearchServiceFilterOptionsTest { + private static final String SOURCE_STREAM = "source-stream-allowed"; + private static final String OTHER_STREAM = "source-stream-denied"; + + @Mock + private MoreSearch moreSearch; + @Mock + private StreamService streamService; + @Mock + private DBEventDefinitionService eventDefinitionService; + @Mock + private ObjectMapper objectMapper; + @Mock + private Subject subject; + + private EventsSearchService service; + + @BeforeEach + void setUp() { + service = new EventsSearchService(moreSearch, streamService, eventDefinitionService, objectMapper); + } + + @Test + void returnsTagsInAggregationOrderForSubjectWithoutGlobalStreamPermission() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"), slice("credential-access"), slice("linux"))); + + final var result = service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + // The aggregation returns the most used values first; that order must be preserved. + assertThat(result.tags()).containsExactly("windows", "credential-access", "linux"); + } + + @Test + void scopesAggregationToTheSourceStreamsTheSubjectMayRead() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + final ArgumentCaptor> eventStreams = ArgumentCaptor.forClass(Set.class); + final ArgumentCaptor sourceStreams = ArgumentCaptor.forClass(SourceStreamFilter.class); + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), eventStreams.capture(), anyString(), + sourceStreams.capture(), eq(EventDto.FIELD_TAGS), any(), anyMap(), eq(50)); + + assertThat(eventStreams.getValue()).containsExactly(DEFAULT_EVENTS_STREAM_ID); + assertThat(sourceStreams.getValue().isAllAllowed()).isFalse(); + assertThat(sourceStreams.getValue().streamIds()).containsExactly(SOURCE_STREAM); + } + + @Test + void passesTheRequestQueryToTheAggregation() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "source_streams:" + SOURCE_STREAM), subject); + + verify(moreSearch).aggregateSlicesForColumn(eq("source_streams:" + SOURCE_STREAM), any(), anySet(), + anyString(), any(), eq(EventDto.FIELD_TAGS), isNull(), anyMap(), anyInt()); + } + + @Test + void buildsAContainsPatternFromTheFieldQuery() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("credential-access"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "Access"), subject); + + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*access.*"), anyMap(), anyInt()); + } + + @Test + void escapesSupplementaryCharactersInTheFieldQueryAsWholeCodePoints() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of()); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "a🦊b"), subject); + + // The surrogate pair must stay together behind a single escape, not become two escaped halves. + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*a\\🦊b.*"), anyMap(), anyInt()); + } + + @Test + void capsTheFieldQueryLength() { + final var request = new EventsFilterOptionsRequest(List.of(), "", "a".repeat(1000), null); + + assertThat(request.fieldQuery()).hasSize(256); + } + + @Test + void appliesRequestDefaults() throws Exception { + final var request = new EventsFilterOptionsRequest(null, null, null, null); + + assertThat(request.fields()).isEmpty(); + assertThat(request.query()).isEmpty(); + assertThat(request.fieldQuery()).isEmpty(); + // The default time range for collecting filter values is the last 30 days. + assertThat(request.timerange()).isEqualTo(RelativeRange.create(30 * 24 * 60 * 60)); + } + + @Test + void escapesRegexMetacharactersInTheFieldQuery() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("t1003.001"))); + + service.filterOptions(request(List.of(EventDto.FIELD_TAGS), "", "T1003.001"), subject); + + verify(moreSearch).aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), + eq(EventDto.FIELD_TAGS), eq(".*t1003\\.001.*"), anyMap(), anyInt()); + } + + @Test + void skipsValuesWithoutATag() { + mockSourceStreamOnlyPermissions(); + mockAggregation(List.of(slice("windows"), slice(null), slice(""))); + + final var result = service.filterOptions(request(List.of(EventDto.FIELD_TAGS), ""), subject); + + assertThat(result.tags()).containsExactly("windows"); + } + + @Test + void omitsFieldsThatWereNotRequested() { + final var result = service.filterOptions(request(List.of(), ""), subject); + + assertThat(result.tags()).isNull(); + verifyNoInteractions(moreSearch); + } + + private void mockSourceStreamOnlyPermissions() { + when(subject.isPermitted(RestPermissions.STREAMS_READ)).thenReturn(false); + when(subject.isPermitted(permission(DEFAULT_EVENTS_STREAM_ID))).thenReturn(false); + when(subject.isPermitted(permission(DEFAULT_SYSTEM_EVENTS_STREAM_ID))).thenReturn(false); + when(subject.isPermitted(permission(SOURCE_STREAM))).thenReturn(true); + when(subject.isPermitted(permission(OTHER_STREAM))).thenReturn(false); + when(streamService.streamAllIds()).thenAnswer(invocation -> java.util.stream.Stream.of(SOURCE_STREAM, OTHER_STREAM)); + } + + private void mockAggregation(List slices) { + when(moreSearch.aggregateSlicesForColumn(anyString(), any(), anySet(), anyString(), any(), anyString(), + any(), anyMap(), anyInt())).thenReturn(slices); + } + + private static EventsFilterOptionsRequest request(List fields, String query) { + return request(fields, query, ""); + } + + private static EventsFilterOptionsRequest request(List fields, String query, String fieldQuery) { + return new EventsFilterOptionsRequest(fields, query, fieldQuery, RelativeRange.allTime()); + } + + private static String permission(String streamId) { + return String.join(":", RestPermissions.STREAMS_READ, streamId); + } + + private static Slice slice(String value) { + return new Slice(value, null, 1, Map.of()); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java b/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java index effa3ef132fe..f15e544029a7 100644 --- a/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java +++ b/graylog2-server/src/test/java/org/graylog/events/search/MoreSearchAdapterAggregationIT.java @@ -34,6 +34,7 @@ public abstract class MoreSearchAdapterAggregationIT extends ElasticsearchBaseTe private static final String INDEX_NAME = "graylog_0"; private static final Set ALL_STREAMS = Set.of("stream-a", "stream-b"); + private static final String EVENTS_STREAM = "000000000000000000000002"; private MoreSearchAdapter adapter; @@ -52,7 +53,7 @@ public void aggregateSlicesForColumn_groupsByField() { final List result = adapter.aggregateSlicesForColumn( "*", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "gl2_source_input", Map.of(), 100); + Map.of(), "gl2_source_input", null, Map.of(), 100); final Map countsByInput = result.stream() .collect(Collectors.toMap(Slice::value, Slice::count)); @@ -68,7 +69,7 @@ public void aggregateSlicesForColumn_withQueryFilter() { final List result = adapter.aggregateSlicesForColumn( "gl2_source_input:input-1", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "streams", Map.of(), 100); + Map.of(), "streams", null, Map.of(), 100); final Map countsByStream = result.stream() .collect(Collectors.toMap(Slice::value, Slice::count)); @@ -83,11 +84,113 @@ public void aggregateSlicesForColumn_emptyResultForNoMatch() { final List result = adapter.aggregateSlicesForColumn( "gl2_source_input:nonexistent", RelativeRange.allTime(), Set.of(INDEX_NAME), ALL_STREAMS, null, allAllowed(), - Map.of(), "streams", Map.of(), 100); + Map.of(), "streams", null, Map.of(), 100); assertThat(result).isEmpty(); } + // --- aggregateSlicesForColumn source stream permission tests --- + // + // The events table and its filter dropdowns rely on the SourceStreamFilter argument alone to keep a + // user from seeing values off events they may not read, so assert it actually filters rather than + // trusting the caller wiring. + + @Test + public void aggregateSlicesForColumn_onlyReturnsValuesFromAllowedSourceStreams() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-a")), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("windows", 2, "execution", 1)); + } + + @Test + public void aggregateSlicesForColumn_returnsValuesForADifferentAllowedSourceStream() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-b")), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("linux", 1, "credential-access", 1)); + } + + @Test + public void aggregateSlicesForColumn_returnsValuesFromEverySourceStreamWhenAllAllowed() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("windows", "execution", "linux", "credential-access", + "t1003.001", "t1003x001", "access-token"); + } + + @Test + public void aggregateSlicesForColumn_returnsNothingForAnEmptySourceStreamAllowList() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of()), + Map.of(), "tags", null, Map.of(), 100); + + assertThat(result).isEmpty(); + } + + // --- aggregateSlicesForColumn bucket pattern tests --- + // + // The bucket pattern backs the server-side search of the tags filter dropdown. It must filter the + // aggregated values themselves (not the events), stay literal for regex metacharacters, and compose + // with the source stream allow-list. + + @Test + public void aggregateSlicesForColumn_bucketPatternOnlyReturnsMatchingValues() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*access.*", Map.of(), 100); + + assertThat(tagCounts(result)) + .containsExactlyInAnyOrderEntriesOf(Map.of("credential-access", 1, "access-token", 1)); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternTreatsEscapedMetacharactersLiterally() { + // An unescaped "." would also match the "t1003x001" tag. + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*t1003\\.001.*", Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("t1003.001"); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternComposesWithSourceStreamAllowList() { + // "access" tags exist in stream-b and stream-c; only the stream-b one may be returned. + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, SourceStreamFilter.allowList(Set.of("stream-b")), + Map.of(), "tags", ".*access.*", Map.of(), 100); + + assertThat(tagCounts(result)).containsOnlyKeys("credential-access"); + } + + @Test + public void aggregateSlicesForColumn_bucketPatternWithoutMatchesReturnsNothing() { + final List result = adapter.aggregateSlicesForColumn( + "", RelativeRange.allTime(), Set.of(INDEX_NAME), + Set.of(EVENTS_STREAM), null, allAllowed(), + Map.of(), "tags", ".*doesnotexist.*", Map.of(), 100); + + assertThat(result).isEmpty(); + } + + private static Map tagCounts(List slices) { + return slices.stream().collect(Collectors.toMap(Slice::value, Slice::count)); + } + // --- aggregateGroupedTerms tests --- @Test diff --git a/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json b/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json index 82e45bc398c4..51ff47502814 100644 --- a/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json +++ b/graylog2-server/src/test/resources/org/graylog/events/search/more_search_adapter_aggregation.json @@ -74,6 +74,66 @@ } } ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-0" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-a", + "timestamp": "2024-01-01 06:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-a"], + "tags": ["windows", "execution"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-1" } }, + { + "data": { + "source": "localhost", + "message": "second event in stream-a", + "timestamp": "2024-01-01 07:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-a"], + "tags": ["windows"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-2" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-b", + "timestamp": "2024-01-01 08:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-b"], + "tags": ["linux", "credential-access"] + } + } + ] + }, + { + "document": [ + { "index": { "indexName": "graylog_0", "indexType": "message", "indexId": "event-3" } }, + { + "data": { + "source": "localhost", + "message": "event in stream-c", + "timestamp": "2024-01-01 09:00:00.000", + "streams": ["000000000000000000000002"], + "source_streams": ["stream-c"], + "tags": ["t1003.001", "t1003x001", "access-token"] + } + } + ] } ] } diff --git a/graylog2-web-interface/src/components/events/TagsFilter.test.tsx b/graylog2-web-interface/src/components/events/TagsFilter.test.tsx index 49dee3b90e27..53374ffd7fca 100644 --- a/graylog2-web-interface/src/components/events/TagsFilter.test.tsx +++ b/graylog2-web-interface/src/components/events/TagsFilter.test.tsx @@ -16,13 +16,14 @@ */ import * as React from 'react'; import { render, screen, waitFor } from 'wrappedTestingLibrary'; +import userEvent from '@testing-library/user-event'; import { Events } from '@graylog/server-api'; import TagsFilter from 'components/events/TagsFilter'; jest.mock('@graylog/server-api', () => ({ - Events: { slices: jest.fn() }, + Events: { filterOptions: jest.fn() }, })); jest.mock('routing/useQuery', () => ({ @@ -30,7 +31,10 @@ jest.mock('routing/useQuery', () => ({ default: () => ({}), })); -const mockedSlices = Events.slices as jest.MockedFunction; +const mockedFilterOptions = Events.filterOptions as jest.MockedFunction; + +const filterOptionsResponse = (tags: Array) => + ({ tags }) as unknown as Awaited>; const tagsAttribute = { id: 'tags', @@ -39,72 +43,74 @@ const tagsAttribute = { filterable: true, } as const; +const renderFilter = () => + render( + t} + onSubmit={jest.fn()} + />, + ); + describe('TagsFilter', () => { beforeEach(() => { - mockedSlices.mockReset(); + mockedFilterOptions.mockReset(); }); - it('queries Events.slices with slice_column=tags and renders returned values', async () => { - mockedSlices.mockResolvedValue({ - slices: [ - { value: 'phishing', count: 5, title: null, meta: {} }, - { value: 'exfil', count: 2, title: null, meta: {} }, - ], - } as unknown as Awaited>); - - render( - t} - onSubmit={jest.fn()} - />, - ); + it('requests the tags filter options and renders returned values', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse(['exfil', 'phishing'])); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalledWith(expect.objectContaining({ slice_column: 'tags', include_all: true })); + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ fields: ['tags'] })); }); expect(await screen.findByText('exfil')).toBeInTheDocument(); expect(await screen.findByText('phishing')).toBeInTheDocument(); }); - it('renders an empty list when slices returns nothing', async () => { - mockedSlices.mockResolvedValue({ slices: [] } as unknown as Awaited>); - - render( - t} - onSubmit={jest.fn()} - />, - ); + it('renders an empty list when no tags are returned', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse([])); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalled(); + expect(mockedFilterOptions).toHaveBeenCalled(); }); expect(screen.queryByText('phishing')).not.toBeInTheDocument(); }); - it('falls back to empty suggestions when the request fails', async () => { - mockedSlices.mockRejectedValue(new Error('boom')); - - render( - t} - onSubmit={jest.fn()} - />, + it('forwards the typed search query to the server', async () => { + mockedFilterOptions.mockResolvedValue(filterOptionsResponse(['credential-access'])); + + renderFilter(); + + await waitFor(() => { + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ field_query: '' })); + }); + + await userEvent.type(await screen.findByPlaceholderText('Search for tags'), 'access'); + + // The search box debounces for 1s before the query is forwarded. + await waitFor( + () => { + expect(mockedFilterOptions).toHaveBeenCalledWith(expect.objectContaining({ field_query: 'access' })); + }, + { timeout: 5000 }, ); + }, 10000); + + it('falls back to empty suggestions when the request fails', async () => { + mockedFilterOptions.mockRejectedValue(new Error('boom')); + + renderFilter(); await waitFor(() => { - expect(mockedSlices).toHaveBeenCalled(); + expect(mockedFilterOptions).toHaveBeenCalled(); }); expect(screen.queryByText('phishing')).not.toBeInTheDocument(); diff --git a/graylog2-web-interface/src/components/events/TagsFilter.tsx b/graylog2-web-interface/src/components/events/TagsFilter.tsx index 2a3ff9eea41a..5475ee5d7a91 100644 --- a/graylog2-web-interface/src/components/events/TagsFilter.tsx +++ b/graylog2-web-interface/src/components/events/TagsFilter.tsx @@ -23,8 +23,6 @@ import { Events } from '@graylog/server-api'; import type { FilterComponentProps } from 'stores/PaginationTypes'; import SuggestionsList from 'components/common/EntityFilters/FilterConfiguration/SuggestionsList'; import useQuery_ from 'routing/useQuery'; -import useDebouncedValue from 'hooks/useDebouncedValue'; -import { MISSING_BUCKET_NAME } from 'views/Constants'; const DEFAULT_SEARCH_PARAMS = { query: '', @@ -39,39 +37,15 @@ const TIMERANGE_30D = { type: 'relative', range: 30 * 24 * 60 * 60 }; // Hard cap on rendered suggestions; relies on the search box to narrow further. const MAX_RENDERED_SUGGESTIONS = 100; -// Debounce window for forwarding the typed prefix to a server-side fetcher. -const PREFIX_DEBOUNCE_MS = 300; - type Suggestion = { id: string; value: string }; -const emptyFilter = { - alerts: 'include' as const, - extra_filters: {}, - aggregation_timerange: { type: 'relative', range: 0 }, - id: [] as string[], - priority: [] as string[], - event_definitions: [] as string[], - key: [] as string[], -}; - -const fetchTagSuggestions = (streamId: string | undefined, _prefix?: string): Promise => - Events.slices({ - include_all: true, - slice_column: 'tags', +const fetchTagSuggestions = (streamId: string | undefined, prefix?: string): Promise => + Events.filterOptions({ + fields: ['tags'], query: streamId ? `source_streams:${streamId}` : '', - filter: emptyFilter, + field_query: prefix ?? '', timerange: TIMERANGE_30D, - }).then((response) => - (response?.slices ?? []) - .map((slice) => slice?.value) - // Drop the scripting-API missing-bucket placeholder — events without tags would otherwise - // surface as a literal "(Empty Value)" suggestion, which isn't a real tag to filter by. - .filter( - (value): value is string => typeof value === 'string' && value.length > 0 && value !== MISSING_BUCKET_NAME, - ) - .sort() - .map((value) => ({ id: value, value })), - ); + }).then((response) => (response?.tags ?? []).map((value) => ({ id: value, value }))); type FetchSuggestions = (streamId: string | undefined, prefix?: string) => Promise; @@ -80,9 +54,10 @@ type FetchSuggestions = (streamId: string | undefined, prefix?: string) => Promi * truth (indexed events vs. defined event definitions) so the fetcher is injected. * * Set `serverSidePrefix` when the backend caps results (e.g. the event-definitions - * suggest endpoint, which is hard-capped at 100). In that mode the debounced search-box - * query is forwarded to the fetcher as a prefix and included in the React Query key so - * later keystrokes can reach entries past the server cap. + * suggest endpoint, which is hard-capped at 100). In that mode the search-box query + * (already debounced by the suggestions list) is forwarded to the fetcher as a prefix + * and included in the React Query key so later keystrokes can reach entries past the + * server cap. */ export const createTagsFilter = ({ queryKeyPrefix, @@ -97,10 +72,9 @@ export const createTagsFilter = ({ }) => { const Filter = ({ attribute, allActiveFilters, filter, filterValueRenderer, onSubmit }: FilterComponentProps) => { const [searchParams, setSearchParams] = useState(DEFAULT_SEARCH_PARAMS); - const [debouncedQuery] = useDebouncedValue(searchParams.query, PREFIX_DEBOUNCE_MS); const { stream_id: streamId } = useQuery_(); const streamIdParam = streamScoped && typeof streamId === 'string' ? streamId : undefined; - const prefixParam = serverSidePrefix ? debouncedQuery : undefined; + const prefixParam = serverSidePrefix ? searchParams.query : undefined; const { data: allSuggestions, isInitialLoading, @@ -149,6 +123,7 @@ const TagsFilter = createTagsFilter({ queryKeyPrefix: ['events', 'tag-suggestions', `t${TIMERANGE_30D.range}`], fetchSuggestions: fetchTagSuggestions, streamScoped: true, + serverSidePrefix: true, }); export default TagsFilter;