Skip to content

Commit feb6e6e

Browse files
at055612at055612
authored
Merge pull request #4136 from gchq/gh-4131-effective-time-format
PR for #4131 - EffectiveTime header variable/stream attribute should allow for missing milli seconds
2 parents 9d3ad60 + c26565b commit feb6e6e

File tree

17 files changed

+986
-105
lines changed

17 files changed

+986
-105
lines changed

Diff for: ‎stroom-app/src/test/java/stroom/test/DataLoader.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import stroom.receive.common.ProgressHandler;
2626
import stroom.receive.common.StreamTargetStreamHandlers;
2727
import stroom.receive.common.StroomStreamProcessor;
28-
import stroom.util.date.DateUtil;
2928
import stroom.util.io.AbstractFileVisitor;
3029
import stroom.util.io.FileUtil;
3130

@@ -134,7 +133,7 @@ public void loadInputStream(final String feedName,
134133

135134
final AttributeMap map = new AttributeMap();
136135
map.put(StandardHeaderArguments.FEED, feedName);
137-
map.put(StandardHeaderArguments.EFFECTIVE_TIME, DateUtil.createNormalDateTimeString(effectiveMs));
136+
map.putDateTime(StandardHeaderArguments.EFFECTIVE_TIME, effectiveMs);
138137

139138
final ProgressHandler progressHandler = new ProgressHandler("Data Loader");
140139
streamTargetStreamHandlers.handle(feedName, null, map, handler -> {
@@ -179,7 +178,7 @@ private void loadZipFile(final Path file, final boolean mandateEffectiveDate, fi
179178
final AttributeMap attributeMap = new AttributeMap();
180179
attributeMap.put(StandardHeaderArguments.FEED, feedName);
181180
attributeMap.put("TestData", "Loaded By SetupSampleData");
182-
attributeMap.put(StandardHeaderArguments.EFFECTIVE_TIME, DateUtil.createNormalDateTimeString(effectiveMs));
181+
attributeMap.putDateTime(StandardHeaderArguments.EFFECTIVE_TIME, effectiveMs);
183182

184183
final ProgressHandler progressHandler = new ProgressHandler("Data Loader");
185184
streamTargetStreamHandlers.handle(feedName, null, attributeMap, handler -> {

Diff for: ‎stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataUploadTaskHandler.java

+2-5
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import stroom.task.api.TaskContextFactory;
2828
import stroom.task.api.TaskProgressHandler;
2929
import stroom.util.EntityServiceExceptionUtil;
30-
import stroom.util.date.DateUtil;
3130
import stroom.util.io.StreamUtil;
3231
import stroom.util.shared.EntityServiceException;
3332

@@ -103,14 +102,12 @@ private void uploadData(final TaskContext taskContext,
103102
}
104103

105104
if (effectiveMs != null) {
106-
attributeMap.put(StandardHeaderArguments.EFFECTIVE_TIME,
107-
DateUtil.createNormalDateTimeString(effectiveMs));
105+
attributeMap.putDateTime(StandardHeaderArguments.EFFECTIVE_TIME, effectiveMs);
108106
}
109107
attributeMap.put(StandardHeaderArguments.REMOTE_FILE, fileName);
110108
attributeMap.put(StandardHeaderArguments.FEED, feedName);
111109
attributeMap.put(StandardHeaderArguments.TYPE, typeName);
112-
attributeMap.put(StandardHeaderArguments.RECEIVED_TIME,
113-
DateUtil.createNormalDateTimeString(System.currentTimeMillis()));
110+
attributeMap.putCurrentDateTime(StandardHeaderArguments.RECEIVED_TIME);
114111
attributeMap.put(StandardHeaderArguments.USER_AGENT, "STROOM-UI");
115112
attributeMap.put("UploadedBy", securityContext.getUserIdentityForAudit());
116113

Diff for: ‎stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMap.java

+51
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package stroom.meta.api;
22

33
import stroom.util.NullSafe;
4+
import stroom.util.date.DateUtil;
45

6+
import java.time.Instant;
57
import java.util.Collection;
68
import java.util.Collections;
79
import java.util.List;
@@ -38,6 +40,55 @@ private AttributeMap(final Builder builder) {
3840
putAll(builder.attributes);
3941
}
4042

43+
@Override
44+
public String put(final String key, final String value) {
45+
if (key != null && StandardHeaderArguments.DATE_HEADER_KEYS.contains(key)) {
46+
final String normalisedValue = DateUtil.normaliseDate(value, true);
47+
return super.put(key, normalisedValue);
48+
} else {
49+
return super.put(key, value);
50+
}
51+
}
52+
53+
/**
54+
* Puts the current date time into the map in Stroom standard ISO 8601 format,
55+
* e.g. {@link DateUtil#createNormalDateTimeString()} using the specified key.
56+
*
57+
* @return The previous value for the key.
58+
*/
59+
public String putCurrentDateTime(final String key) {
60+
// Already normalised, so use super.put not the local one
61+
return super.put(key, DateUtil.createNormalDateTimeString());
62+
}
63+
64+
/**
65+
* Puts the specified date time (as epoch millis) into the map in Stroom standard ISO 8601 format,
66+
* e.g. {@link DateUtil#createNormalDateTimeString()} using the specified key.
67+
*
68+
* @return The previous value for the key.
69+
*/
70+
public String putDateTime(final String key, final Long epochMs) {
71+
final String dateStr = DateUtil.createNormalDateTimeString(epochMs);
72+
// Already normalised, so use super.put not the local one
73+
return super.put(key, dateStr);
74+
}
75+
76+
/**
77+
* Puts the specified {@link Instant} into the map in Stroom standard ISO 8601 format,
78+
* e.g. {@link DateUtil#createNormalDateTimeString()} using the specified key.
79+
*
80+
* @return The previous value for the key.
81+
*/
82+
public String putDateTime(final String key, final Instant instant) {
83+
if (instant == null) {
84+
return super.put(key, null);
85+
} else {
86+
final String dateStr = DateUtil.createNormalDateTimeString(instant.toEpochMilli());
87+
// Already normalised, so use super.put not the local one
88+
return super.put(key, dateStr);
89+
}
90+
}
91+
4192
/**
4293
* Put an entry where the value is itself a collection of values, e.g. a list of files
4394
*/

Diff for: ‎stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMapUtil.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
import stroom.util.NullSafe;
2020
import stroom.util.cert.CertificateExtractor;
21-
import stroom.util.date.DateUtil;
2221
import stroom.util.io.StreamUtil;
2322

2423
import jakarta.servlet.http.HttpServletRequest;
@@ -36,6 +35,7 @@
3635
import java.nio.charset.Charset;
3736
import java.nio.charset.StandardCharsets;
3837
import java.security.cert.X509Certificate;
38+
import java.time.Instant;
3939
import java.time.LocalDateTime;
4040
import java.time.ZoneOffset;
4141
import java.time.format.DateTimeFormatter;
@@ -245,8 +245,8 @@ private static void addAllSecureTokens(final HttpServletRequest httpServletReque
245245
}
246246

247247
if (cert.getNotAfter() != null) {
248-
final String remoteCertExpiry = DateUtil.createNormalDateTimeString(cert.getNotAfter().getTime());
249-
attributeMap.put(StandardHeaderArguments.REMOTE_CERT_EXPIRY, remoteCertExpiry);
248+
final long timeEpochMs = cert.getNotAfter().getTime();
249+
attributeMap.putDateTime(StandardHeaderArguments.REMOTE_CERT_EXPIRY, timeEpochMs);
250250
} else {
251251
LOGGER.debug("Cert {} doesn't have a Not After date", cert);
252252
}
@@ -289,10 +289,10 @@ private static void putHeader(final String headerToken,
289289
if (CERT_EXPIRY_HEADER_TOKEN.equals(headerToken)) {
290290
try {
291291
final LocalDateTime localDateTime = LocalDateTime.parse(headerValue, CERT_EXPIRY_DATE_FORMATTER);
292-
final String newHeaderValue = DateUtil.createNormalDateTimeString(
293-
localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli());
294-
LOGGER.debug("Converting certificate expiry date from [{}] to [{}]", headerValue, newHeaderValue);
295-
attributeMap.put(StandardHeaderArguments.REMOTE_CERT_EXPIRY, newHeaderValue);
292+
final Instant instant = localDateTime.toInstant(ZoneOffset.UTC);
293+
LOGGER.debug("Converting certificate expiry date from [{}] to [{}]", headerValue, instant);
294+
attributeMap.putDateTime(StandardHeaderArguments.REMOTE_CERT_EXPIRY, instant);
295+
296296
} catch (Exception e) {
297297
LOGGER.error("Unable to create header {} from header {} with value [{}].",
298298
StandardHeaderArguments.REMOTE_CERT_EXPIRY, CERT_EXPIRY_HEADER_TOKEN, headerValue, e);

Diff for: ‎stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/StandardHeaderArguments.java

+7
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,11 @@ public interface StandardHeaderArguments {
6666
"transfer-encoding",
6767
"expect",
6868
COMPRESSION);
69+
70+
/**
71+
* Header keys for values that are date/time strings
72+
*/
73+
Set<String> DATE_HEADER_KEYS = Set.of(
74+
EFFECTIVE_TIME,
75+
RECEIVED_TIME);
6976
}

Diff for: ‎stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMap.java

+103-17
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@
33
import stroom.test.common.TestUtil;
44

55
import io.vavr.Tuple;
6-
import org.assertj.core.api.Assertions;
76
import org.junit.jupiter.api.DynamicTest;
87
import org.junit.jupiter.api.Test;
98
import org.junit.jupiter.api.TestFactory;
109

1110
import java.io.ByteArrayInputStream;
1211
import java.io.IOException;
1312
import java.nio.charset.StandardCharsets;
13+
import java.time.Duration;
14+
import java.time.Instant;
1415
import java.util.Arrays;
1516
import java.util.Collections;
1617
import java.util.HashSet;
@@ -208,7 +209,7 @@ void testEquality1() {
208209
"FOO", "123",
209210
"BAR", "456"));
210211

211-
Assertions.assertThat(attributeMap1)
212+
assertThat(attributeMap1)
212213
.isEqualTo(attributeMap2);
213214
}
214215

@@ -223,7 +224,7 @@ void testEquality2() {
223224
"FOO", "123",
224225
"BAR", "456"));
225226

226-
Assertions.assertThat(attributeMap1)
227+
assertThat(attributeMap1)
227228
.isNotEqualTo(attributeMap2);
228229
}
229230

@@ -239,7 +240,7 @@ void testEquality3() {
239240
"bar", "VALUE2"));
240241

241242
// Value cases not same
242-
Assertions.assertThat(attributeMap1)
243+
assertThat(attributeMap1)
243244
.isNotEqualTo(attributeMap2);
244245
}
245246

@@ -330,33 +331,118 @@ Stream<DynamicTest> testContainsValue() {
330331
void testPut() {
331332
final AttributeMap attributeMap1 = new AttributeMap();
332333

333-
Assertions.assertThat(attributeMap1)
334+
assertThat(attributeMap1)
334335
.isEmpty();
335336

336337
attributeMap1.put("foo", "value1a");
337-
Assertions.assertThat(attributeMap1)
338+
assertThat(attributeMap1)
338339
.hasSize(1);
339-
Assertions.assertThat(attributeMap1.get("Foo"))
340+
assertThat(attributeMap1.get("Foo"))
340341
.isEqualTo("value1a");
341342

342343
attributeMap1.put("FOO", "value1b"); // 'same' key, new val
343-
Assertions.assertThat(attributeMap1)
344+
assertThat(attributeMap1)
344345
.hasSize(1);
345-
Assertions.assertThat(attributeMap1.get("Foo"))
346+
assertThat(attributeMap1.get("Foo"))
346347
.isEqualTo("value1b");
347348

348349
attributeMap1.put("bar", "value2a");
349-
Assertions.assertThat(attributeMap1)
350+
assertThat(attributeMap1)
350351
.hasSize(2);
351-
Assertions.assertThat(attributeMap1.get("BAR"))
352+
assertThat(attributeMap1.get("BAR"))
352353
.isEqualTo("value2a");
353354
}
354355

356+
@Test
357+
void testPut_withDateNormalisation() {
358+
final AttributeMap attributeMap1 = new AttributeMap();
359+
final String dateStrIn = "2010-01-01T23:59:59.123456+00:00";
360+
final String dateStrOut = "2010-01-01T23:59:59.123Z";
361+
362+
for (final String key : StandardHeaderArguments.DATE_HEADER_KEYS) {
363+
attributeMap1.clear();
364+
assertThat(attributeMap1)
365+
.isEmpty();
366+
367+
attributeMap1.put(key, dateStrIn);
368+
369+
assertThat(attributeMap1)
370+
.hasSize(1);
371+
372+
assertThat(attributeMap1.get(key))
373+
.isEqualTo(dateStrOut);
374+
}
375+
}
376+
377+
@Test
378+
void testPutDateTime1() {
379+
final AttributeMap attributeMap1 = new AttributeMap();
380+
final String dateStrIn = "2010-01-01T23:59:59.123456+00:00";
381+
final String dateStrOut = "2010-01-01T23:59:59.123Z";
382+
final long epochMs = Instant.parse(dateStrIn).toEpochMilli();
383+
final String key = "foo";
384+
385+
assertThat(attributeMap1)
386+
.isEmpty();
387+
388+
attributeMap1.putDateTime(key, epochMs);
389+
390+
assertThat(attributeMap1)
391+
.hasSize(1);
392+
393+
assertThat(attributeMap1.get(key))
394+
.isEqualTo(dateStrOut);
395+
}
396+
397+
@Test
398+
void testPutDateTime2() {
399+
final AttributeMap attributeMap1 = new AttributeMap();
400+
final String dateStrIn = "2010-01-01T23:59:59.123456+00:00";
401+
final String dateStrOut = "2010-01-01T23:59:59.123Z";
402+
final Instant instant = Instant.parse(dateStrIn);
403+
final String key = "foo";
404+
405+
assertThat(attributeMap1)
406+
.isEmpty();
407+
408+
attributeMap1.putDateTime(key, instant);
409+
410+
assertThat(attributeMap1)
411+
.hasSize(1);
412+
413+
assertThat(attributeMap1.get(key))
414+
.isEqualTo(dateStrOut);
415+
}
416+
417+
@Test
418+
void testPutCurrentDateTime() {
419+
final AttributeMap attributeMap1 = new AttributeMap();
420+
final String dateStrIn = "2010-01-01T23:59:59.123456+00:00";
421+
final String dateStrOut = "2010-01-01T23:59:59.123Z";
422+
final String key = "foo";
423+
424+
assertThat(attributeMap1)
425+
.isEmpty();
426+
427+
final Instant now = Instant.now();
428+
attributeMap1.putCurrentDateTime(key);
429+
430+
assertThat(attributeMap1)
431+
.hasSize(1);
432+
433+
final String val = attributeMap1.get(key);
434+
assertThat(val)
435+
.isNotNull();
436+
final Instant instant = Instant.parse(val);
437+
assertThat(Duration.between(now, instant))
438+
.isLessThan(Duration.ofMillis(100));
439+
}
440+
355441
@Test
356442
void testComputeIfAbsent1() {
357443

358444
final AttributeMap attributeMap1 = new AttributeMap();
359-
Assertions.assertThat(attributeMap1)
445+
assertThat(attributeMap1)
360446
.isEmpty();
361447
final AtomicInteger callCount = new AtomicInteger();
362448

@@ -365,9 +451,9 @@ void testComputeIfAbsent1() {
365451
return "value(" + k + ")";
366452
});
367453

368-
Assertions.assertThat(computedVal)
454+
assertThat(computedVal)
369455
.isEqualTo("value(foo)");
370-
Assertions.assertThat(callCount)
456+
assertThat(callCount)
371457
.hasValue(1);
372458
}
373459

@@ -376,7 +462,7 @@ void testComputeIfAbsent2() {
376462

377463
final AttributeMap attributeMap1 = new AttributeMap();
378464
attributeMap1.put("foo", "value(initial)");
379-
Assertions.assertThat(attributeMap1)
465+
assertThat(attributeMap1)
380466
.hasSize(1);
381467
final AtomicInteger callCount = new AtomicInteger();
382468

@@ -385,9 +471,9 @@ void testComputeIfAbsent2() {
385471
return "value(" + k + ")";
386472
});
387473

388-
Assertions.assertThat(computedVal)
474+
assertThat(computedVal)
389475
.isEqualTo("value(initial)");
390-
Assertions.assertThat(callCount)
476+
assertThat(callCount)
391477
.hasValue(0);
392478
}
393479
}

0 commit comments

Comments
 (0)