Skip to content

Commit 084b725

Browse files
stroomdev66stroomdev66
authored
Merge pull request #4807 from gchq/gh-4795_http_appender_props
#4795 Add missing properties to HTTPAppender
2 parents 7a64aa6 + cba5d11 commit 084b725

File tree

5 files changed

+150
-47
lines changed

5 files changed

+150
-47
lines changed

Diff for: ‎stroom-pipeline/src/main/java/stroom/pipeline/writer/HTTPAppender.java

+94-33
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,10 @@ public class HTTPAppender extends AbstractAppender {
109109
private final ErrorReceiverProxy errorReceiverProxy;
110110

111111
private String forwardUrl;
112+
private Long timeout;
112113
private Long connectionTimeout;
113-
private Long readTimeout;
114+
private Long connectionRequestTimeout;
115+
private Long timeToLive;
114116
private Long forwardChunkSize;
115117
private Set<String> metaKeySet = getMetaKeySet(META_KEYS_DEFAULT);
116118

@@ -174,11 +176,17 @@ protected Output createOutput() {
174176
final HttpClientConfiguration.Builder builder = HttpClientConfiguration.builder();
175177
builder.tlsConfiguration(httpTlsConfiguration);
176178

179+
if (timeout != null) {
180+
builder.timeout(StroomDuration.ofMillis(timeout.intValue()));
181+
}
177182
if (connectionTimeout != null) {
178183
builder.connectionTimeout(StroomDuration.ofMillis(connectionTimeout.intValue()));
179184
}
180-
if (readTimeout != null) {
181-
builder.connectionRequestTimeout(StroomDuration.ofMillis(readTimeout.intValue()));
185+
if (connectionRequestTimeout != null) {
186+
builder.connectionRequestTimeout(StroomDuration.ofMillis(connectionRequestTimeout.intValue()));
187+
}
188+
if (timeToLive != null) {
189+
builder.timeToLive(StroomDuration.ofMillis(timeToLive.intValue()));
182190
}
183191

184192
final HttpClientConfiguration httpClientConfiguration = builder.build();
@@ -501,14 +509,24 @@ private void addRequestProperty(final HttpUriRequestBase request,
501509
}
502510

503511
private void logConnectionToDebug() {
504-
LOGGER.debug(() -> LogUtil.message("About to connect to {} with requestMethod: {}, contentType: {}, " +
505-
"readTimeout: {}, connectionTimeout: {}, useCompression: {}, " +
506-
"compressionMethod: {}, forwardChunkSize: {}, request properties:\n{}",
512+
LOGGER.debug(() -> LogUtil.message("About to connect to {} with " +
513+
"requestMethod: {}, " +
514+
"contentType: {}, " +
515+
"timeout: {}, " +
516+
"connectionTimeout: {}, " +
517+
"connectionRequestTimeout: {}, " +
518+
"timeToLive: {}, " +
519+
"useCompression: {}, " +
520+
"compressionMethod: {}, " +
521+
"forwardChunkSize: {}, " +
522+
"request properties:\n{}",
507523
forwardUrl,
508524
requestMethod,
509525
contentType,
510-
readTimeout,
526+
timeout,
511527
connectionTimeout,
528+
connectionRequestTimeout,
529+
timeToLive,
512530
outputStreamSupport.isUseCompression(),
513531
outputStreamSupport.getCompressionMethod(),
514532
forwardChunkSize,
@@ -628,39 +646,82 @@ public void setForwardUrl(final String forwardUrl) {
628646
this.forwardUrl = forwardUrl;
629647
}
630648

631-
@PipelineProperty(description = "How long to wait before we abort sending data due to connection timeout. " +
649+
@PipelineProperty(description = "Determines the timeout until arrival of a response from the opposite endpoint. " +
650+
"A timeout value of zero is interpreted as an infinite timeout. " +
651+
"Default: 3 minutes. " +
632652
"The timeout is specified as either milliseconds, e.g. '60000' or with a " +
633653
"duration suffix, e.g. '500ms', '2s', '1m', etc.",
634654
displayPriority = 6)
655+
public void setTimeout(final String string) {
656+
timeout = null;
657+
if (NullSafe.isNonEmptyString(string)) {
658+
timeout = ModelStringUtil.parseDurationString(string);
659+
}
660+
}
661+
662+
@PipelineProperty(description = "Determines the timeout until a new connection is fully established. " +
663+
"This may also include transport security negotiation exchanges such as SSL or " +
664+
"TLS protocol negotiation. " +
665+
"A timeout value of zero is interpreted as an infinite timeout. " +
666+
"Default: 3 minutes. " +
667+
"The timeout is specified as either milliseconds, e.g. '60000' or with a " +
668+
"duration suffix, e.g. '500ms', '2s', '1m', etc.",
669+
displayPriority = 7)
635670
public void setConnectionTimeout(final String string) {
636671
connectionTimeout = null;
637672
if (NullSafe.isNonEmptyString(string)) {
638673
connectionTimeout = ModelStringUtil.parseDurationString(string);
639674
}
640675
}
641676

677+
@PipelineProperty(description = "Returns the connection lease request timeout used when requesting a connection " +
678+
"from the connection manager. " +
679+
"Default: 3 minutes. " +
680+
"The timeout is specified as either milliseconds, e.g. '60000' or with a " +
681+
"duration suffix, e.g. '500ms', '2s', '1m', etc.",
682+
displayPriority = 8)
683+
public void setConnectionRequestTimeout(final String string) {
684+
connectionRequestTimeout = null;
685+
if (NullSafe.isNonEmptyString(string)) {
686+
connectionRequestTimeout = ModelStringUtil.parseDurationString(string);
687+
}
688+
}
689+
642690
@PipelineProperty(description = "How long to wait for data to be available before closing the connection. " +
643691
"The timeout is specified as either milliseconds, e.g. '60000' or with a " +
644692
"duration suffix, e.g. '500ms', '2s', '1m', etc.",
645-
displayPriority = 7)
693+
displayPriority = 8)
694+
@Deprecated
646695
public void setReadTimeout(final String string) {
647-
readTimeout = null;
696+
if (string != null) {
697+
setConnectionRequestTimeout(string);
698+
}
699+
}
700+
701+
@PipelineProperty(description = "The maximum time a pooled connection can stay idle (not leased to any thread) " +
702+
"before it is shut down. " +
703+
"Default: 1 hour. " +
704+
"The timeout is specified as either milliseconds, e.g. '60000' or with a " +
705+
"duration suffix, e.g. '500ms', '2s', '1m', etc.",
706+
displayPriority = 9)
707+
public void setTimeToLive(final String string) {
708+
timeToLive = null;
648709
if (NullSafe.isNonEmptyString(string)) {
649-
readTimeout = ModelStringUtil.parseDurationString(string);
710+
timeToLive = ModelStringUtil.parseDurationString(string);
650711
}
651712
}
652713

653714
@PipelineProperty(description = "Should data be sent in chunks and if so how big should the chunks be. " +
654715
"Size is either specified in bytes e.g. '1024' or with a IEC unit suffix, " +
655716
"e.g. '1K', '1M', '1G', etc.",
656-
displayPriority = 8)
717+
displayPriority = 10)
657718
public void setForwardChunkSize(final String string) {
658719
this.forwardChunkSize = ModelStringUtil.parseIECByteSizeString(string);
659720
}
660721

661722
@PipelineProperty(description = "Should data be compressed when sending",
662723
defaultValue = DEFAULT_USE_COMPRESSION_PROP_VALUE,
663-
displayPriority = 9)
724+
displayPriority = 11)
664725
public void setUseCompression(final boolean useCompression) {
665726
outputStreamSupport.setUseCompression(useCompression);
666727
}
@@ -672,7 +733,7 @@ public void setUseCompression(final boolean useCompression) {
672733
"other destinations, but is only applicable for compression types " +
673734
"'gz', 'zstd' or 'deflate'.",
674735
defaultValue = DEFAULT_USE_CONTENT_ENCODING_PROP_VALUE,
675-
displayPriority = 10)
736+
displayPriority = 12)
676737
public void setUseContentEncodingHeader(final boolean useContentEncodingHeader) {
677738
this.useContentEncodingHeader = useContentEncodingHeader;
678739
}
@@ -681,7 +742,7 @@ public void setUseContentEncodingHeader(final boolean useContentEncodingHeader)
681742
description = "Compression method to apply, if compression is enabled. Supported values: " +
682743
CompressionUtil.SUPPORTED_COMPRESSORS + ".",
683744
defaultValue = DEFAULT_COMPRESSION_METHOD_PROP_VALUE,
684-
displayPriority = 11)
745+
displayPriority = 13)
685746
public void setCompressionMethod(final String compressionMethod) {
686747
try {
687748
outputStreamSupport.setCompressionMethod(compressionMethod);
@@ -695,7 +756,7 @@ public void setCompressionMethod(final String compressionMethod) {
695756
description = "Specifies Which meta data keys will have their values logged in the send log. A Comma " +
696757
"delimited string of keys.",
697758
defaultValue = META_KEYS_DEFAULT,
698-
displayPriority = 12)
759+
displayPriority = 14)
699760
public void setLogMetaKeys(final String string) {
700761
metaKeySet = getMetaKeySet(string);
701762
}
@@ -706,85 +767,85 @@ public void setLogMetaKeys(final String string) {
706767
"Set this to false if you are explicitly setting key/trust store properties on " +
707768
"this HttpAppender.",
708769
defaultValue = "true",
709-
displayPriority = 13)
770+
displayPriority = 15)
710771
@Deprecated
711772
public void setUseJvmSslConfig(final boolean useJvmSslConfig) {
712773
// this.useJvmSslConfig = useJvmSslConfig;
713774
}
714775

715776
@PipelineProperty(description = "The key store file path on the server",
716-
displayPriority = 14)
777+
displayPriority = 16)
717778
public void setKeyStorePath(final String keyStorePath) {
718779
sslConfigBuilder.withKeyStorePath(keyStorePath);
719780
}
720781

721782
@PipelineProperty(description = "The key store type. " +
722783
"Valid values are ['JCEKS', 'JKS', 'DKS', 'PKCS11', 'PKCS12'].",
723784
defaultValue = "JKS",
724-
displayPriority = 15)
785+
displayPriority = 17)
725786
public void setKeyStoreType(final String keyStoreType) {
726787
sslConfigBuilder.withKeyStoreType(keyStoreType);
727788
}
728789

729790
@PipelineProperty(description = "The key store password",
730-
displayPriority = 16)
791+
displayPriority = 18)
731792
public void setKeyStorePassword(final String keyStorePassword) {
732793
sslConfigBuilder.withKeyStorePassword(keyStorePassword);
733794
}
734795

735796
@PipelineProperty(description = "The trust store file path on the server",
736-
displayPriority = 17)
797+
displayPriority = 19)
737798
public void setTrustStorePath(final String trustStorePath) {
738799
sslConfigBuilder.withTrustStorePath(trustStorePath);
739800
}
740801

741802
@PipelineProperty(description = "The trust store type " +
742803
"Valid values are ['JCEKS', 'JKS', 'DKS', 'PKCS11', 'PKCS12'].",
743804
defaultValue = "JKS",
744-
displayPriority = 18)
805+
displayPriority = 20)
745806
public void setTrustStoreType(final String trustStoreType) {
746807
sslConfigBuilder.withTrustStoreType(trustStoreType);
747808
}
748809

749810
@PipelineProperty(description = "The trust store password",
750-
displayPriority = 19)
811+
displayPriority = 21)
751812
public void setTrustStorePassword(final String trustStorePassword) {
752813
sslConfigBuilder.withTrustStorePassword(trustStorePassword);
753814
}
754815

755816
@PipelineProperty(description = "Set this to true to verify that the destination host name matches against " +
756817
"the host names in the certificate supplied by the destination server.",
757818
defaultValue = "true",
758-
displayPriority = 20)
819+
displayPriority = 22)
759820
public void setHostnameVerificationEnabled(final boolean hostnameVerificationEnabled) {
760821
sslConfigBuilder.withHostnameVerificationEnabled(hostnameVerificationEnabled);
761822
}
762823

763824
@PipelineProperty(description = "The SSL protocol to use",
764825
defaultValue = "TLSv1.2",
765-
displayPriority = 21)
826+
displayPriority = 23)
766827
public void setSslProtocol(final String sslProtocol) {
767828
sslConfigBuilder.withSslProtocol(NullSafe.get(sslProtocol, String::trim));
768829
}
769830

770831
@PipelineProperty(description = "The HTTP request method. Valid values are " +
771832
"GET, POST, HEAD, OPTIONS, PUT, DELETE and TRACE.",
772833
defaultValue = DEFAULT_REQUEST_METHOD_PROP_VALUE,
773-
displayPriority = 22)
834+
displayPriority = 24)
774835
public void setRequestMethod(String requestMethod) {
775836
this.requestMethod = NullSafe.get(requestMethod, String::trim, String::toUpperCase);
776837
}
777838

778839
@PipelineProperty(description = "The content type",
779840
defaultValue = "application/json",
780-
displayPriority = 23)
841+
displayPriority = 25)
781842
public void setContentType(String contentType) {
782843
this.contentType = contentType;
783844
}
784845

785846
@PipelineProperty(description = "Provide stream metadata as HTTP headers",
786847
defaultValue = "true",
787-
displayPriority = 24)
848+
displayPriority = 26)
788849
public void setHttpHeadersIncludeStreamMetaData(final boolean newValue) {
789850
this.httpHeadersIncludeStreamMetaData = newValue;
790851
}
@@ -795,7 +856,7 @@ public void setHttpHeadersIncludeStreamMetaData(final boolean newValue) {
795856
"If httpHeadersStreamMetaDataAllowList contains keys, " +
796857
"httpHeadersStreamMetaDataDenyList is ignored.",
797858
defaultValue = "",
798-
displayPriority = 25)
859+
displayPriority = 27)
799860
public void setHttpHeadersStreamMetaDataAllowList(final String newValue) {
800861
this.httpHeadersStreamMetaDataAllowList = newValue;
801862
}
@@ -806,25 +867,25 @@ public void setHttpHeadersStreamMetaDataAllowList(final String newValue) {
806867
"If httpHeadersStreamMetaDataAllowList contains keys, " +
807868
"httpHeadersStreamMetaDataDenyList is ignored.",
808869
defaultValue = "",
809-
displayPriority = 26)
870+
displayPriority = 28)
810871
public void setHttpHeadersStreamMetaDataDenyList(final String newValue) {
811872
this.httpHeadersStreamMetaDataDenyList = newValue;
812873
}
813874

814875
@PipelineProperty(description = "Additional HTTP Header 1, format is 'HeaderName: HeaderValue'",
815-
displayPriority = 27)
876+
displayPriority = 29)
816877
public void setHttpHeadersUserDefinedHeader1(final String headerText) {
817878
this.httpHeadersUserDefinedHeader1 = headerText;
818879
}
819880

820881
@PipelineProperty(description = "Additional HTTP Header 2, format is 'HeaderName: HeaderValue'",
821-
displayPriority = 28)
882+
displayPriority = 30)
822883
public void setHttpHeadersUserDefinedHeader2(final String headerText) {
823884
this.httpHeadersUserDefinedHeader2 = headerText;
824885
}
825886

826887
@PipelineProperty(description = "Additional HTTP Header 3, format is 'HeaderName: HeaderValue'",
827-
displayPriority = 29)
888+
displayPriority = 31)
828889
public void setHttpHeadersUserDefinedHeader3(final String headerText) {
829890
this.httpHeadersUserDefinedHeader3 = headerText;
830891
}

Diff for: ‎stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/ForwardHttpPostConfig.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ public class ForwardHttpPostConfig extends AbstractConfig implements IsProxyConf
2121
private static final StroomDuration DEFAULT_FORWARD_DELAY = StroomDuration.ZERO;
2222
private static final Integer DEFAULT_MAX_RETRIES = 3;
2323
private static final StroomDuration DEFAULT_RETRY_DELAY = StroomDuration.ofSeconds(10);
24-
private static final StroomDuration DEFAULT_FORWARD_TIMEOUT = StroomDuration.ofMinutes(1);
2524

2625
private final boolean enabled;
2726
private final boolean instant;
@@ -80,10 +79,10 @@ public ForwardHttpPostConfig(@JsonProperty("enabled") final boolean enabled,
8079
private HttpClientConfiguration createDefaultHttpClientConfiguration() {
8180
return HttpClientConfiguration
8281
.builder()
83-
.timeout(DEFAULT_FORWARD_TIMEOUT)
84-
.connectionTimeout(DEFAULT_FORWARD_TIMEOUT)
85-
.connectionRequestTimeout(DEFAULT_FORWARD_TIMEOUT)
86-
.timeToLive(DEFAULT_FORWARD_TIMEOUT)
82+
.timeout(StroomDuration.ofMinutes(1))
83+
.connectionTimeout(StroomDuration.ofMinutes(1))
84+
.connectionRequestTimeout(StroomDuration.ofMinutes(1))
85+
.timeToLive(StroomDuration.ofHours(1))
8786
.build();
8887
}
8988

Diff for: ‎stroom-proxy/stroom-proxy-app/src/test/resources/stroom/dist/proxy-expected.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ proxyConfig:
5050
maxConnectionsPerRoute: 1024
5151
proxy: null
5252
retries: 0
53-
timeToLive: "PT1M"
53+
timeToLive: "PT1H"
5454
timeout: "PT1M"
5555
tls: null
5656
userAgent: null

0 commit comments

Comments
 (0)