Skip to content

Commit b79ee69

Browse files
dennisoelkersdennisoelkers
authored
Merge branch 'master' into refactor/fail-lock-acquisition-if-factory-holds-one
2 parents 3738500 + eb5853b commit b79ee69

89 files changed

Lines changed: 2066 additions & 695 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎changelog/unreleased/issue-24781.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "fixed"
2+
message = "Fix overly strict parsing of escaped characters in pipeline function parse_json."
3+
4+
issues = ["24781"]
5+
pulls = ["24785"]

‎changelog/unreleased/pr-24659.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "f"
2+
message = "Input stop/start button UX fixes"
3+
4+
issues = ["24235", "graylog-plugin-enterprise#12659"]
5+
pulls = ["24659"]

‎changelog/unreleased/pr-24755.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "fixed"
2+
message = "Clean up documents for failed job triggers."
3+
4+
issues = ["23884"]
5+
pulls = ["24755"]

‎data-node/src/main/resources/opensearch/config/jvm.options‎

Lines changed: 5 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,9 @@
11
## JVM configuration
22

3-
################################################################
4-
## IMPORTANT: JVM heap size
5-
################################################################
6-
##
7-
## You should always set the min and max JVM heap
8-
## size to the same value. For example, to set
9-
## the heap to 4 GB, set:
10-
##
11-
## -Xms4g
12-
## -Xmx4g
13-
##
14-
## See https://opensearch.org/docs/opensearch/install/important-settings/
15-
## for more information
16-
##
17-
################################################################
18-
19-
# Xms represents the initial size of total heap space
20-
# Xmx represents the maximum size of total heap space
21-
22-
-Xms1g
23-
-Xmx1g
24-
25-
################################################################
26-
## Expert settings
27-
################################################################
28-
##
29-
## All settings below this section are considered
30-
## expert settings. Don't tamper with them unless
31-
## you understand what you are doing
32-
##
33-
################################################################
34-
35-
## GC configuration
36-
8-10:-XX:+UseConcMarkSweepGC
37-
8-10:-XX:CMSInitiatingOccupancyFraction=75
38-
8-10:-XX:+UseCMSInitiatingOccupancyOnly
39-
403
## G1GC Configuration
41-
# NOTE: G1 GC is only supported on JDK version 10 or later
42-
# to use G1GC, uncomment the next two lines and update the version on the
43-
# following three lines to your version of the JDK
44-
# 10:-XX:-UseConcMarkSweepGC
45-
# 10:-XX:-UseCMSInitiatingOccupancyOnly
46-
11-:-XX:+UseG1GC
47-
11-:-XX:G1ReservePercent=25
48-
11-:-XX:InitiatingHeapOccupancyPercent=30
4+
-XX:+UseG1GC
5+
-XX:G1ReservePercent=25
6+
-XX:InitiatingHeapOccupancyPercent=30
497

508
## JVM temporary directory
519
-Djava.io.tmpdir=${OPENSEARCH_TMPDIR}
@@ -63,18 +21,8 @@
6321
# specify an alternative path for JVM fatal error logs
6422
-XX:ErrorFile=/tmp/hs_err_pid%p.log
6523

66-
## JDK 8 GC logging
67-
8:-XX:+PrintGCDetails
68-
8:-XX:+PrintGCDateStamps
69-
8:-XX:+PrintTenuringDistribution
70-
8:-XX:+PrintGCApplicationStoppedTime
71-
8:-Xloggc:/tmp/gc.log
72-
8:-XX:+UseGCLogFileRotation
73-
8:-XX:NumberOfGCLogFiles=32
74-
8:-XX:GCLogFileSize=64m
75-
7624
# JDK 9+ GC logging
77-
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/tmp/gc.log:utctime,pid,tags:filecount=32,filesize=64m
25+
-Xlog:gc*,gc+age=trace,safepoint:file=/tmp/gc.log:utctime,pid,tags:filecount=32,filesize=64m
7826

7927
# Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380)
80-
18-:-Djava.security.manager=allow
28+
-Djava.security.manager=allow

‎graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
import com.fasterxml.jackson.databind.ObjectMapper;
2121
import com.github.joschi.jadconfig.util.Duration;
2222
import com.google.common.annotations.VisibleForTesting;
23-
import com.google.common.collect.Streams;
2423
import io.opentelemetry.instrumentation.annotations.WithSpan;
2524
import jakarta.inject.Inject;
2625
import jakarta.inject.Named;
26+
import org.graylog.plugins.views.search.errors.SearchTypeErrorParser;
2727
import org.graylog.shaded.elasticsearch7.org.apache.http.ContentTooLongException;
2828
import org.graylog.shaded.elasticsearch7.org.apache.http.client.config.RequestConfig;
2929
import org.graylog.shaded.elasticsearch7.org.elasticsearch.ElasticsearchException;
@@ -46,6 +46,7 @@
4646
import org.graylog2.indexer.MapperParsingException;
4747
import org.graylog2.indexer.MasterNotDiscoveredException;
4848
import org.graylog2.indexer.ParentCircuitBreakingException;
49+
import org.graylog2.indexer.exceptions.ResultWindowLimitExceededException;
4950
import org.slf4j.Logger;
5051
import org.slf4j.LoggerFactory;
5152

@@ -55,7 +56,6 @@
5556
import java.util.Optional;
5657
import java.util.regex.Matcher;
5758
import java.util.regex.Pattern;
58-
import java.util.stream.Collectors;
5959

6060
import static com.google.common.base.Preconditions.checkArgument;
6161

@@ -97,21 +97,6 @@ public SearchResponse search(SearchRequest searchRequest, String errorMessage) {
9797
return firstResponseFrom(result, errorMessage);
9898
}
9999

100-
public List<MultiSearchResponse.Item> msearch(List<SearchRequest> searchRequests, String errorMessage) {
101-
final MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
102-
103-
indexerMaxConcurrentSearches.ifPresent(multiSearchRequest::maxConcurrentSearchRequests);
104-
indexerMaxConcurrentShardRequests.ifPresent(maxShardRequests -> searchRequests
105-
.forEach(request -> request.setMaxConcurrentShardRequests(maxShardRequests)));
106-
107-
searchRequests.forEach(multiSearchRequest::add);
108-
109-
final MultiSearchResponse result = this.execute((c, requestOptions) -> c.msearch(multiSearchRequest, requestOptions), errorMessage);
110-
111-
return Streams.stream(result)
112-
.collect(Collectors.toList());
113-
}
114-
115100
private SearchResponse firstResponseFrom(MultiSearchResponse result, String errorMessage) {
116101
checkArgument(result != null);
117102
checkArgument(result.getResponses().length == 1);
@@ -184,6 +169,10 @@ private RequestOptions requestOptions() {
184169

185170
public static RuntimeException exceptionFrom(Exception e, String errorMessage) {
186171
if (e instanceof ElasticsearchException elasticsearchException) {
172+
final Integer resultWindowLimitFromError = SearchTypeErrorParser.getResultWindowLimitFromError(elasticsearchException);
173+
if (resultWindowLimitFromError != null) {
174+
throw new ResultWindowLimitExceededException(resultWindowLimitFromError);
175+
}
187176
if (isIndexNotFoundException(elasticsearchException)) {
188177
return IndexNotFoundException.create(errorMessage + elasticsearchException.getResourceId(), elasticsearchException.getIndex().getName());
189178
}

‎graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
import com.fasterxml.jackson.databind.ObjectMapper;
2121
import com.github.joschi.jadconfig.util.Duration;
2222
import com.google.common.annotations.VisibleForTesting;
23-
import com.google.common.collect.Streams;
2423
import io.opentelemetry.instrumentation.annotations.WithSpan;
2524
import jakarta.inject.Inject;
2625
import jakarta.inject.Named;
26+
import org.graylog.plugins.views.search.errors.SearchTypeErrorParser;
2727
import org.graylog.shaded.opensearch2.org.apache.http.ContentTooLongException;
2828
import org.graylog.shaded.opensearch2.org.apache.http.client.config.RequestConfig;
2929
import org.graylog.shaded.opensearch2.org.opensearch.OpenSearchException;
@@ -47,6 +47,7 @@
4747
import org.graylog2.indexer.MapperParsingException;
4848
import org.graylog2.indexer.MasterNotDiscoveredException;
4949
import org.graylog2.indexer.ParentCircuitBreakingException;
50+
import org.graylog2.indexer.exceptions.ResultWindowLimitExceededException;
5051
import org.slf4j.Logger;
5152
import org.slf4j.LoggerFactory;
5253

@@ -56,7 +57,6 @@
5657
import java.util.Optional;
5758
import java.util.regex.Matcher;
5859
import java.util.regex.Pattern;
59-
import java.util.stream.Collectors;
6060

6161
import static com.google.common.base.Preconditions.checkArgument;
6262

@@ -98,21 +98,6 @@ public SearchResponse search(SearchRequest searchRequest, String errorMessage) {
9898
return firstResponseFrom(result, errorMessage);
9999
}
100100

101-
public List<MultiSearchResponse.Item> msearch(List<SearchRequest> searchRequests, String errorMessage) {
102-
var multiSearchRequest = new MultiSearchRequest();
103-
104-
indexerMaxConcurrentSearches.ifPresent(multiSearchRequest::maxConcurrentSearchRequests);
105-
indexerMaxConcurrentShardRequests.ifPresent(maxShardRequests -> searchRequests
106-
.forEach(request -> request.setMaxConcurrentShardRequests(maxShardRequests)));
107-
108-
searchRequests.forEach(multiSearchRequest::add);
109-
110-
final MultiSearchResponse result = this.execute((c, requestOptions) -> c.msearch(multiSearchRequest, requestOptions), errorMessage);
111-
112-
return Streams.stream(result)
113-
.collect(Collectors.toList());
114-
}
115-
116101
private SearchResponse firstResponseFrom(MultiSearchResponse result, String errorMessage) {
117102
checkArgument(result != null);
118103
checkArgument(result.getResponses().length == 1);
@@ -185,6 +170,10 @@ private RequestOptions requestOptions() {
185170

186171
public static RuntimeException exceptionFrom(Exception e, String errorMessage) {
187172
if (e instanceof OpenSearchException openSearchException) {
173+
final Integer resultWindowLimitFromError = SearchTypeErrorParser.getResultWindowLimitFromError(openSearchException);
174+
if (resultWindowLimitFromError != null) {
175+
throw new ResultWindowLimitExceededException(resultWindowLimitFromError);
176+
}
188177
if (isIndexNotFoundException(openSearchException)) {
189178
return IndexNotFoundException.create(errorMessage + openSearchException.getResourceId(), openSearchException.getIndex().getName());
190179
}

‎graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@
114114
import static org.graylog2.notifications.Notification.Type.REMOTE_REINDEX_FINISHED;
115115
import static org.graylog2.notifications.Notification.Type.REMOTE_REINDEX_RUNNING;
116116

117+
@Deprecated(forRemoval = true)
117118
@Singleton
118119
public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigrationAdapter {
119120
private static final Logger LOG = LoggerFactory.getLogger(RemoteReindexingMigrationAdapterOS2.class);

‎graylog-storage-opensearch3/src/main/java/org/graylog/storage/opensearch3/OpenSearchClient.java‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
import com.fasterxml.jackson.databind.ObjectMapper;
2121
import com.github.joschi.jadconfig.util.Duration;
2222
import com.google.common.annotations.VisibleForTesting;
23-
import com.google.common.collect.Streams;
2423
import io.opentelemetry.instrumentation.annotations.WithSpan;
2524
import jakarta.inject.Inject;
2625
import jakarta.inject.Named;
26+
import org.graylog.plugins.views.search.errors.SearchTypeErrorParser;
2727
import org.graylog.shaded.opensearch2.org.apache.http.ContentTooLongException;
2828
import org.graylog.shaded.opensearch2.org.apache.http.client.config.RequestConfig;
2929
import org.graylog.shaded.opensearch2.org.opensearch.OpenSearchException;
@@ -47,6 +47,7 @@
4747
import org.graylog2.indexer.MapperParsingException;
4848
import org.graylog2.indexer.MasterNotDiscoveredException;
4949
import org.graylog2.indexer.ParentCircuitBreakingException;
50+
import org.graylog2.indexer.exceptions.ResultWindowLimitExceededException;
5051
import org.slf4j.Logger;
5152
import org.slf4j.LoggerFactory;
5253

@@ -56,7 +57,6 @@
5657
import java.util.Optional;
5758
import java.util.regex.Matcher;
5859
import java.util.regex.Pattern;
59-
import java.util.stream.Collectors;
6060

6161
import static com.google.common.base.Preconditions.checkArgument;
6262

@@ -99,21 +99,6 @@ public SearchResponse search(SearchRequest searchRequest, String errorMessage) {
9999
return firstResponseFrom(result, errorMessage);
100100
}
101101

102-
public List<MultiSearchResponse.Item> msearch(List<SearchRequest> searchRequests, String errorMessage) {
103-
var multiSearchRequest = new MultiSearchRequest();
104-
105-
indexerMaxConcurrentSearches.ifPresent(multiSearchRequest::maxConcurrentSearchRequests);
106-
indexerMaxConcurrentShardRequests.ifPresent(maxShardRequests -> searchRequests
107-
.forEach(request -> request.setMaxConcurrentShardRequests(maxShardRequests)));
108-
109-
searchRequests.forEach(multiSearchRequest::add);
110-
111-
final MultiSearchResponse result = this.execute((c, requestOptions) -> c.msearch(multiSearchRequest, requestOptions), errorMessage);
112-
113-
return Streams.stream(result)
114-
.collect(Collectors.toList());
115-
}
116-
117102
private SearchResponse firstResponseFrom(MultiSearchResponse result, String errorMessage) {
118103
checkArgument(result != null);
119104
checkArgument(result.getResponses().length == 1);
@@ -186,6 +171,10 @@ private RequestOptions requestOptions() {
186171

187172
public static RuntimeException exceptionFrom(Exception e, String errorMessage) {
188173
if (e instanceof OpenSearchException openSearchException) {
174+
final Integer resultWindowLimitFromError = SearchTypeErrorParser.getResultWindowLimitFromError(openSearchException);
175+
if (resultWindowLimitFromError != null) {
176+
throw new ResultWindowLimitExceededException(resultWindowLimitFromError);
177+
}
189178
if (isIndexNotFoundException(openSearchException)) {
190179
return IndexNotFoundException.create(errorMessage + openSearchException.getResourceId(), openSearchException.getIndex().getName());
191180
}

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineRulesMetadataDao.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import com.google.auto.value.AutoValue;
2323
import org.graylog2.database.BuildableMongoEntity;
2424

25+
import java.util.HashMap;
2526
import java.util.HashSet;
27+
import java.util.Map;
2628
import java.util.Set;
2729

2830
@JsonDeserialize(builder = AutoValue_PipelineRulesMetadataDao.Builder.class)
@@ -33,6 +35,8 @@ public abstract class PipelineRulesMetadataDao implements BuildableMongoEntity<P
3335
private static final String FIELD_STREAMS = "streams";
3436
private static final String FIELD_FUNCTIONS = "functions";
3537
private static final String FIELD_DEPRECATED_FUNCTIONS = "deprecated_functions";
38+
public static final String FIELD_ROUTING_RULES = "routing_rules";
39+
public static final String FIELD_ROUTED_STREAMS = "routed_streams";
3640
public static final String FIELD_HAS_INPUT_REFERENCES = "has_input_references";
3741

3842
@JsonProperty(FIELD_PIPELINE_ID)
@@ -50,6 +54,14 @@ public abstract class PipelineRulesMetadataDao implements BuildableMongoEntity<P
5054
@JsonProperty(FIELD_DEPRECATED_FUNCTIONS)
5155
public abstract Set<String> deprecatedFunctions();
5256

57+
@JsonProperty(FIELD_ROUTING_RULES)
58+
// Maps rule ID to IDs of routed stream
59+
public abstract Map<String, Set<String>> streamsByRuleId();
60+
61+
@JsonProperty(FIELD_ROUTED_STREAMS)
62+
// Maps stream ID to stream title
63+
public abstract Map<String, String> routedStreamTitleById();
64+
5365
@JsonProperty(FIELD_HAS_INPUT_REFERENCES)
5466
public abstract Boolean hasInputReferences();
5567

@@ -65,7 +77,9 @@ public static Builder builder() {
6577
.streams(new HashSet<>())
6678
.functions(new HashSet<>())
6779
.deprecatedFunctions(new HashSet<>())
68-
.hasInputReferences(false);
80+
.hasInputReferences(false)
81+
.streamsByRuleId(new HashMap<>())
82+
.routedStreamTitleById(new HashMap<>());
6983
}
7084

7185
@AutoValue.Builder
@@ -86,6 +100,12 @@ public abstract static class Builder implements BuildableMongoEntity.Builder<Pip
86100
@JsonProperty(FIELD_DEPRECATED_FUNCTIONS)
87101
public abstract Builder deprecatedFunctions(Set<String> deprecatedFunctions);
88102

103+
@JsonProperty(FIELD_ROUTING_RULES)
104+
public abstract Builder streamsByRuleId(Map<String, Set<String>> routingRules);
105+
106+
@JsonProperty(FIELD_ROUTED_STREAMS)
107+
public abstract Builder routedStreamTitleById(Map<String, String> routedStreamsMap);
108+
89109
@JsonProperty(FIELD_HAS_INPUT_REFERENCES)
90110
public abstract Builder hasInputReferences(Boolean hasInputReferences);
91111
}

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineMetadataService.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ public Map<String, PipelineRulesMetadataDao> get(Set<String> pipelineIds) {
7575
.collect(Collectors.toMap(PipelineRulesMetadataDao::pipelineId, dao -> dao));
7676
}
7777

78+
public Set<PipelineRulesMetadataDao> getRoutingPipelines(String streamId) {
79+
return collection.find(Filters.exists(PipelineRulesMetadataDao.FIELD_ROUTED_STREAMS + "." + streamId, true))
80+
.into(new HashSet<>());
81+
}
82+
7883
public Set<String> getPipelinesByRule(final String ruleId) {
7984
return collection.find(eq(PipelineRulesMetadataDao.FIELD_RULES, ruleId))
8085
.map(PipelineRulesMetadataDao::pipelineId)

0 commit comments

Comments
 (0)