Merge branch 'master' into gh-4610-ref-auto-purge

Author: at055612

CHANGELOG.md

@@ -13,6 +13,55 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 ~~~
 
 
+## [v7.9-beta.8] - 2025-04-09
+
+* Uplift BCrypt lib to 0.4.3.
+
+* Add BCrypt as a hashing algorithm to data feed keys. Change Data feed key auth to require the header as configured by `dataFeedKeyOwnerMetaKey`. Change `hashAlgorithmId` to `hashAlgorithm` in the data feed keys json file.
+
+
+## [v7.9-beta.7] - 2025-04-07
+
+* Issue **#4831** : Fix Data Retention -> Impact Summary not showing any data.
+
+* Issue **#4829** : Fix stuck searches.
+
+* Issue **#4830** : Use a cache rather than sessions to maintain auth flow state to avoid creating unnecessary sessions.
+
+* Issue **#4842** : Fix null session when doing OIDC code flow with KeyCloak.
+
+* Issue **#4844** : Fix issue where vis parent table filters are not applied to the right data values.
+
+* Issue **#4837** : Change the fetching of OIDC config to use jersey client instead of Apache http client. The yaml properties `appConfig.security.authentication.openId.httpClient` and `proxyConfig.security.authentication.openId.httpClient` have been removed. Configuration of the jersey client is now done using `jerseyClients.OPEN_ID.` (see https://gchq.github.io/stroom-docs/docs/install-guide/configuration/stroom-and-proxy/common-configuration/#jersey-http-client-configuration).
+
+* Issue **#4849** : Fix the default forwarding queue config so that it retries for HTTP and not for FILE. Add the config prop `queue.queueAndRetryEnabled` to control whether forwarding is queued with retry handling or not. Add the config prop `atomicMoveEnabled` to `forwardFileDestinations` items to allow disabling of atomic file moves when using a remote file system that doesn't support atomic moves.
+
+
+## [v7.9-beta.6] - 2025-04-07
+
+* Issue **#4109** : Add `receive` config properties `x509CertificateHeader`, `x509CertificateDnHeader` and `allowedCertificateProviders` to control the use of certificates and DNs placed in the request headers by load balancers or reverse proxies that are doing the TLS termination. Header keys were previously hard coded. `allowedCertificateProviders` is an allow list of FQDN/IPs that are allowed to use the cert/DN headers.
+
+* Add Dropwizard Metrics to proxy.
+
+* Change proxy to use the same caching as stroom.
+
+* Remove unused proxy config property `maxAggregateAge`. `aggregationFrequency` controls the aggregation age/frequency.
+
+* Stroom-Proxy instances that are making remote feed status requests using an API key or token, will now need to hold the application permission `Check Receipt Status` in Stroom. This prevents anybody with an API key from checking feed statuses.
+
+* Issue **#4312** : Add Data Feed Keys to proxy and stroom to allow their use in data receipt authentication. Replace `proxyConfig.receive.(certificateAuthenticationEnabled|tokenAuthenticationEnabled)` with `proxyConfig.receive.enabledAuthenticationTypes` that takes values: `DATA_FEED_KEY|TOKEN|CERTIFICATE` (where `TOKEN` means an oauth token or an API key). The feed status check endpoint `/api/feedStatus/v1` has been deprecated. Proxies with a version >=v7.9 should now use `/api/feedStatus/v2`.
+
+* Replace proxy config prop `proxyConfig.eventStore.maxOpenFiles` with `proxyConfig.eventStore.openFilesCache`.
+
+* Add optional auto-generation of the `Feed` attribute using property `proxyConfig.receive.feedNameGenerationEnabled`. This is used alongside properties `proxyConfig.receive.feedNameTemplate` (which defines a template for the auto-generated feed name using meta keys and their values) and `feedNameGenerationMandatoryHeaders` which defines the mandatory meta headers that must be present for a auto-generation of the feed name to be possible.
+
+* Add a new _Content Templates_ screen to stroom (requires `Manage Content Templates` application permission). This screen is used to define rules for matching incoming data where the feed does not exist and creating content to process data for that feed.
+
+* Feed status check calls made by a proxy into stroom now require the application permission `Check Receipt Status`. This is to stop anyone with an API key from discovering the feeds available in stroom. Any existing API keys used for feed status checks on proxy will need to have `Check Receipt Status` granted to the owner of the key.
+
+* Issue **#4844** : Fix issue where vis parent table filters are not applied to the right data values.
+
+
 ## [v7.9-beta.5] - 2025-04-02
 
 * Issue **#4831** : Fix Data Retention -> Impact Summary not showing any data.
@@ -1329,7 +1378,10 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 * Issue **#3830** : Add S3 data storage option.
 
 
-[Unreleased]: https://github.com/gchq/stroom/compare/v7.9-beta.5...HEAD
+[Unreleased]: https://github.com/gchq/stroom/compare/v7.9-beta.8...HEAD
+[v7.9-beta.8]: https://github.com/gchq/stroom/compare/v7.9-beta.7...v7.9-beta.8
+[v7.9-beta.7]: https://github.com/gchq/stroom/compare/v7.9-beta.6...v7.9-beta.7
+[v7.9-beta.6]: https://github.com/gchq/stroom/compare/v7.9-beta.5...v7.9-beta.6
 [v7.9-beta.5]: https://github.com/gchq/stroom/compare/v7.9-beta.4...v7.9-beta.5
 [v7.9-beta.4]: https://github.com/gchq/stroom/compare/v7.9-beta.3...v7.9-beta.4
 [v7.9-beta.3]: https://github.com/gchq/stroom/compare/v7.9-beta.2...v7.9-beta.3

gradle/libs.versions.toml

@@ -36,7 +36,7 @@ aws-crt = { module = "software.amazon.awssdk.crt:aws-crt", version = "0.29.22" }
 aws-s3-transfer-manager = { module = "software.amazon.awssdk:s3-transfer-manager" } # version controlled by AWS BOM
 aws-sqs = { module = "software.amazon.awssdk:sqs" } # version controlled by AWS BOM
 aws-sts = { module = "software.amazon.awssdk:sts" } # version controlled by AWS BOM
-bcrypt = { module = "de.svenkubiak:jBCrypt", version = "0.4.1" }
+bcrypt = { module = "de.svenkubiak:jBCrypt", version = "0.4.3" }
 bouncy-castle = { module = "org.bouncycastle:bcprov-jdk18on", version = "1.78.1" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine" } # version controlled by dropwizard-dependencies
 commons-beanutils = { module = "commons-beanutils:commons-beanutils", version = "1.9.4" }

stroom-config/stroom-config-app/src/test/resources/stroom/config/app/expected.yaml

@@ -882,7 +882,6 @@ appConfig:
         clientSecret: null
         expectedSignerPrefixes: []
         formTokenRequest: true
-        httpClient: null
         identityProviderType: "INTERNAL_IDP"
         issuer: null
         jwksUri: null

stroom-pipeline/docs/ref-data-design.md

@@ -0,0 +1,130 @@
+# Reference Data Load/Lookup Design
+
+## LMDB DB Structure
+
+### ProcessingInfoDb
+
+**Key** - `RefStreamDefinition`
+
+```text
+< pipe doc UUID >< pipe version UUID >< stream ID    >< part index   >
+< ? bytes       >< ? bytes           >< 8 bytes long >< 8 bytes long >
+```
+
+**Value** - `RefDataProcessingInfo`
+
+```text
+< create time ms >< last access time ms >< effective time ms >< processing state ID >
+< 8 bytes long   >< 8 bytes long        >< 8 bytes long      >< 1 byte              >
+```
+
+One entry per ref data stream.
+
+
+### MapUidForwardDb
+
+**Key** - `KeyValueStoreKey`
+
+```text
+< pipe doc UUID >< pipe version UUID >< stream ID    >< part index   >< map name       >
+< ? bytes       >< ? bytes           >< 8 bytes long >< 8 bytes long >< ? bytes String >
+```
+
+**Value** - `ValueStoreKey`
+
+```text
+< map UID >
+< 4 bytes >
+```
+
+One entry per ref data stream and map name.
+
+
+### MapUidReverseDb
+
+**Key** - `KeyValueStoreKey`
+
+```text
+< map UID >
+< 4 bytes >
+```
+
+**Value** - `ValueStoreKey`
+
+```text
+< pipe doc UUID >< pipe version UUID >< stream ID    >< part index   >< map name       >
+< ? bytes       >< ? bytes           >< 8 bytes long >< 8 bytes long >< ? bytes String >
+```
+
+One entry per ref data stream and map name.
+
+
+### KeyValueStoreDb
+
+**Key** - `KeyValueStoreKey`
+
+```text
+< map UID     >< reference data key >
+< 4 bytes UID >< ? bytes String     >
+```
+
+**Value** - `ValueStoreKey`
+
+```text
+< value hash code >< unique ID     >
+< 8 bytes long    >< 2 bytes short >
+```
+
+One entry per key/value ref data entry.
+
+
+### RangeStoreDb
+
+**Key** - `RangeStoreKey`
+
+```text
+< map UID     >< rangeStartInc >< rangeEndExc  >
+< 4 bytes UID >< 8 bytes long  >< 8 bytes long >
+```
+
+**Value** - `ValueStoreKey`
+
+See above
+
+One entry per range/value ref data entry.
+
+
+### ValueStoreDb
+
+**Key** - `ValueStoreKey`
+
+See above
+
+
+**Value** - `RefDataValue`
+
+```text
+< reference data value       >
+< ? bytes string/fastInfoset >
+```
+
+One entry per distinct reference data value, i.e. de-duplicates values.
+
+
+### ValueStoreMetaDb
+
+**Key** - `ValueStoreKey`
+
+See above
+
+
+**Value** - `ValueStoreMeta`
+
+```text
+< type ID >< reference count >
+< 1 byte  >< 3 bytes UnsignedBytes >
+```
+
+One entry per entry in ValueStoreDb.
+Holds the value type information (string/fastInfoset) and the reference count of key/range entries that reference it.
+

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/ProxyOpenIdConfig.java

@@ -2,7 +2,6 @@
 
 import stroom.security.openid.api.AbstractOpenIdConfig;
 import stroom.security.openid.api.IdpType;
-import stroom.util.http.HttpClientConfiguration;
 import stroom.util.shared.IsProxyConfig;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -42,8 +41,7 @@ public ProxyOpenIdConfig(
             @JsonProperty("validIssuers") final Set<String> validIssuers,
             @JsonProperty("uniqueIdentityClaim") final String uniqueIdentityClaim,
             @JsonProperty("userDisplayNameClaim") final String userDisplayNameClaim,
-            @JsonProperty(PROP_NAME_EXPECTED_SIGNER_PREFIXES) final Set<String> expectedSignerPrefixes,
-            @JsonProperty("httpClient") final HttpClientConfiguration httpClient) {
+            @JsonProperty(PROP_NAME_EXPECTED_SIGNER_PREFIXES) final Set<String> expectedSignerPrefixes) {
 
         super(identityProviderType,
                 openIdConfigurationEndpoint,
@@ -62,8 +60,7 @@ public ProxyOpenIdConfig(
                 validIssuers,
                 uniqueIdentityClaim,
                 userDisplayNameClaim,
-                expectedSignerPrefixes,
-                httpClient);
+                expectedSignerPrefixes);
     }
 
     @JsonIgnore
@@ -113,7 +110,6 @@ public ProxyOpenIdConfig withIdentityProviderType(final IdpType identityProvider
                 getValidIssuers(),
                 getUniqueIdentityClaim(),
                 getUserDisplayNameClaim(),
-                getExpectedSignerPrefixes(),
-                getHttpClient());
+                getExpectedSignerPrefixes());
     }
 }

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/ForwardException.java

@@ -3,6 +3,7 @@
 import stroom.meta.api.AttributeMap;
 import stroom.meta.api.StandardHeaderArguments;
 import stroom.proxy.StroomStatusCode;
+import stroom.proxy.app.handler.HttpSender.ResponseStatus;
 import stroom.util.shared.NullSafe;
 
 import java.util.Objects;
@@ -12,15 +13,18 @@ public class ForwardException extends RuntimeException {
     private final StroomStatusCode stroomStatusCode;
     private final String feedName;
     private final boolean isRecoverable;
+    private final int httpResponseCode; // In the case of UNKNOWN_ERROR, this may differ from 999
 
     private ForwardException(final StroomStatusCode stroomStatusCode,
                              final AttributeMap attributeMap,
                              final String message,
+                             final int httpResponseCode,
                              final boolean isRecoverable,
                              final Throwable cause) {
         super(message, cause);
         this.isRecoverable = isRecoverable;
         this.stroomStatusCode = stroomStatusCode;
+        this.httpResponseCode = httpResponseCode;
         this.feedName = NullSafe.get(
                 attributeMap,
                 attrMap -> attrMap.get(StandardHeaderArguments.FEED));
@@ -30,26 +34,52 @@ public static ForwardException recoverable(final StroomStatusCode stroomStatusCo
                                                final AttributeMap attributeMap,
                                                final String message,
                                                final Throwable cause) {
-        return new ForwardException(stroomStatusCode, attributeMap, message, true, cause);
+        return new ForwardException(
+                stroomStatusCode,
+                attributeMap,
+                message,
+                stroomStatusCode.getHttpCode(),
+                true,
+                cause);
     }
 
-    public static ForwardException recoverable(final StroomStatusCode stroomStatusCode,
+    public static ForwardException recoverable(final ResponseStatus responseStatus,
                                                final AttributeMap attributeMap) {
-        Objects.requireNonNull(stroomStatusCode);
-        return new ForwardException(stroomStatusCode, attributeMap, stroomStatusCode.getMessage(), true, null);
+        Objects.requireNonNull(responseStatus);
+        final StroomStatusCode stroomStatusCode = responseStatus.stroomStatusCode();
+        return new ForwardException(
+                stroomStatusCode,
+                attributeMap,
+                Objects.requireNonNullElse(responseStatus.message(), stroomStatusCode.getMessage()),
+                responseStatus.httpResponseCode(),
+                true,
+                null);
     }
 
     public static ForwardException nonRecoverable(final StroomStatusCode stroomStatusCode,
                                                   final AttributeMap attributeMap,
                                                   final String message,
                                                   final Throwable cause) {
-        return new ForwardException(stroomStatusCode, attributeMap, message, false, cause);
+        return new ForwardException(
+                stroomStatusCode,
+                attributeMap,
+                message,
+                stroomStatusCode.getHttpCode(),
+                false,
+                cause);
     }
 
-    public static ForwardException nonRecoverable(final StroomStatusCode stroomStatusCode,
+    public static ForwardException nonRecoverable(final ResponseStatus responseStatus,
                                                   final AttributeMap attributeMap) {
-        Objects.requireNonNull(stroomStatusCode);
-        return new ForwardException(stroomStatusCode, attributeMap, stroomStatusCode.getMessage(), false, null);
+        Objects.requireNonNull(responseStatus);
+        final StroomStatusCode stroomStatusCode = responseStatus.stroomStatusCode();
+        return new ForwardException(
+                stroomStatusCode,
+                attributeMap,
+                Objects.requireNonNullElse(responseStatus.message(), stroomStatusCode.getMessage()),
+                responseStatus.httpResponseCode(),
+                false,
+                null);
     }
 
     public boolean isRecoverable() {
@@ -59,4 +89,8 @@ public boolean isRecoverable() {
     public String getFeedName() {
         return feedName;
     }
+
+    public int getHttpResponseCode() {
+        return httpResponseCode;
+    }
 }