Skip to content

Commit 86e2d08

Browse files
at055612at055612
authored
Merge pull request #4792 from gchq/gh-4700-proxy-failure-handling
PR for #4700 - Stroom Proxy Failed Forwarding
2 parents b4629e6 + 87d5fd0 commit 86e2d08

File tree

123 files changed

+9215
-1371
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

123 files changed

+9215
-1371
lines changed

Diff for: ‎gradle/libs.versions.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ commons-lang = { module = "org.apache.commons:commons-lang3" } # version control
4949
commons-pool2 = { module = "org.apache.commons:commons-pool2", version = "2.12.0" }
5050
commons-text = { module = "org.apache.commons:commons-text" } # version controlled by dropwizard-dependencies
5151
classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.179" }
52-
data-faker = { module = "net.datafaker:datafaker", version = "2.3.0" }
52+
data-faker = { module = "net.datafaker:datafaker", version = "2.4.2" }
5353
dropwizard-assets = { module = "io.dropwizard:dropwizard-assets" } # version controlled by dropwizard-dependencies
5454
dropwizard-auth = { module = "io.dropwizard:dropwizard-auth" } # version controlled by dropwizard-dependencies
5555
dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version.ref = "dropwizard" }

Diff for: ‎stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ReportExecutor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ private String getFileName(final String baseName,
393393
String fileName = baseName;
394394
fileName = NON_BASIC_CHARS.matcher(fileName).replaceAll("");
395395
fileName = MULTIPLE_SPACE.matcher(fileName).replaceAll(" ");
396-
fileName = fileName.replace(" ", "_");
396+
fileName = fileName.replace(' ', '_');
397397
fileName = fileName + "." + extension;
398398
return fileName;
399399
}

Diff for: ‎stroom-app/src/test/java/stroom/test/SampleDataGenerator.java

+78-33
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import stroom.test.common.data.DataWriter;
66
import stroom.test.common.data.FlatDataWriterBuilder;
77
import stroom.test.common.data.XmlAttributesDataWriterBuilder;
8+
import stroom.util.concurrent.ThreadUtil;
89
import stroom.util.io.FileUtil;
10+
import stroom.util.logging.DurationTimer;
911
import stroom.util.logging.LogUtil;
1012

1113
import io.vavr.Tuple;
@@ -44,6 +46,7 @@ public class SampleDataGenerator {
4446
private final Path templatesDir;
4547

4648
private final List<CompletableFuture<Void>> futures = new ArrayList<>();
49+
private final AtomicInteger taskCounter = new AtomicInteger();
4750

4851
@Inject
4952
public SampleDataGenerator() {
@@ -68,37 +71,53 @@ public static void main(String[] args) {
6871
public void generateData(final Path dir) {
6972

7073
ensureAndCleanDir(dir);
74+
LOGGER.info("Cleaned dir {}", dir);
7175

7276
generateDataViewingData(dir);
77+
LOGGER.info("Generated data viewing data");
7378

7479
generateRefDataForEffectiveDateTesting(dir);
80+
LOGGER.info("Generated reference data");
7581

7682
generateCharsetData(dir);
77-
78-
LOGGER.info("Waiting for {} async tasks to complete", futures.size());
83+
LOGGER.info("Generated charset data");
84+
85+
while (true) {
86+
final int runningTasks = taskCounter.get();
87+
if (runningTasks == 0) {
88+
break;
89+
} else {
90+
LOGGER.info("Waiting for {} async tasks to complete", runningTasks);
91+
ThreadUtil.sleep(2_000);
92+
}
93+
}
94+
LOGGER.info("Joining threads");
7995
CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join();
8096
LOGGER.info("Completed generation");
8197
}
8298

83-
private void generateDataViewingData(final Path dir) {
84-
final int shortLoremText = 4;
85-
final int longLoremText = 200;
86-
// Increment the random seed each time so each data set has different but predictable data
87-
long randomSeed = 0;
88-
89-
final DataWriter csvDataWriter = FlatDataWriterBuilder.builder()
90-
.delimitedBy(",")
91-
.enclosedBy("\"")
92-
.outputHeaderRow(true)
93-
.build();
94-
95-
final DataWriter xmlDataWriter = XmlAttributesDataWriterBuilder.builder()
99+
private DataWriter buildXmlDataWriter() {
100+
return XmlAttributesDataWriterBuilder.builder()
96101
.namespace("records:2")
97102
.rootElementName("records")
98103
.recordElementName("record")
99104
.fieldValueElementName("data")
100105
.build();
106+
}
101107

108+
private DataWriter buildCsvDataWriter() {
109+
return FlatDataWriterBuilder.builder()
110+
.delimitedBy(",")
111+
.enclosedBy("\"")
112+
.outputHeaderRow(true)
113+
.build();
114+
}
115+
116+
private void generateDataViewingData(final Path dir) {
117+
final int shortLoremText = 4;
118+
final int longLoremText = 200;
119+
// Increment the random seed each time so each data set has different but predictable data
120+
long randomSeed = 0;
102121

103122
// Data that has one record per line
104123
// One with long lines, one with short
@@ -107,7 +126,7 @@ private void generateDataViewingData(final Path dir) {
107126
1,
108127
"DATA_VIEWING_MULTI_LINE-EVENTS",
109128
"\n",
110-
csvDataWriter,
129+
DataWriterMode.CSV,
111130
shortLoremText,
112131
LocalDateTime.of(2020, 6, 1, 0, 0),
113132
randomSeed++);
@@ -117,7 +136,7 @@ private void generateDataViewingData(final Path dir) {
117136
2,
118137
"DATA_VIEWING_MULTI_LINE-EVENTS",
119138
"\n",
120-
csvDataWriter,
139+
DataWriterMode.CSV,
121140
longLoremText,
122141
LocalDateTime.of(2020, 7, 1, 0, 0),
123142
randomSeed++);
@@ -128,7 +147,7 @@ private void generateDataViewingData(final Path dir) {
128147
1,
129148
"DATA_VIEWING_SINGLE_LINE-EVENTS",
130149
"|",
131-
csvDataWriter,
150+
DataWriterMode.CSV,
132151
shortLoremText,
133152
LocalDateTime.of(2020, 8, 1, 0, 0),
134153
randomSeed++);
@@ -139,7 +158,7 @@ private void generateDataViewingData(final Path dir) {
139158
1,
140159
"DATA_VIEWING_XML_SINGLE_LINE-EVENTS",
141160
"",
142-
xmlDataWriter,
161+
DataWriterMode.XML,
143162
shortLoremText,
144163
LocalDateTime.of(2020, 9, 1, 0, 0),
145164
randomSeed++);
@@ -151,7 +170,7 @@ private void generateDataViewingData(final Path dir) {
151170
1,
152171
"DATA_VIEWING_XML_MULTI_LINE-EVENTS",
153172
"\n",
154-
xmlDataWriter,
173+
DataWriterMode.XML,
155174
shortLoremText,
156175
LocalDateTime.of(2020, 10, 1, 0, 0),
157176
randomSeed++);
@@ -161,25 +180,28 @@ private void generateDataViewingData(final Path dir) {
161180
2,
162181
"DATA_VIEWING_XML_MULTI_LINE-EVENTS",
163182
"\n",
164-
xmlDataWriter,
183+
DataWriterMode.XML,
165184
longLoremText,
166185
LocalDateTime.of(2020, 11, 1, 0, 0),
167186
randomSeed++);
168-
169187
}
170188

171189
private void generateDataViewRawData(final Path dir,
172190
final int fileNo,
173191
final String feedName,
174192
final String recordSeparator,
175-
final DataWriter dataWriter,
193+
final DataWriterMode dataWriterMode,
176194
final int loremWordCount,
177195
final LocalDateTime startDate,
178196
final long randomSeed) {
179197
final Path file = makeInputFilePath(dir, fileNo, feedName);
198+
final DataWriter dataWriter = switch (dataWriterMode) {
199+
case CSV -> buildCsvDataWriter();
200+
case XML -> buildXmlDataWriter();
201+
};
180202

181-
futures.add(CompletableFuture.runAsync(() -> {
182-
LOGGER.info("Generating file {}", file.toAbsolutePath().normalize());
203+
runAsync(() -> {
204+
final DurationTimer timer = DurationTimer.start();
183205
DataGenerator.buildDefinition()
184206
.addFieldDefinition(DataGenerator.randomDateTimeField(
185207
"dateTime",
@@ -201,7 +223,7 @@ private void generateDataViewRawData(final Path dir,
201223
faker -> faker.name().lastName()))
202224
.addFieldDefinition(DataGenerator.fakerField(
203225
"username",
204-
faker -> faker.name().username()))
226+
faker -> faker.internet().username()))
205227
.addFieldDefinition(DataGenerator.fakerField(
206228
"bloodGroup",
207229
faker -> faker.bloodtype().bloodGroup()))
@@ -225,7 +247,9 @@ private void generateDataViewRawData(final Path dir,
225247
.rowCount(2_000)
226248
.withRandomSeed(randomSeed)
227249
.generate();
228-
}));
250+
LOGGER.info("Generated file {} in {}",
251+
file.toAbsolutePath().normalize(), timer);
252+
});
229253
}
230254

231255
private void generateRefDataForEffectiveDateTesting(final Path dir) {
@@ -242,7 +266,7 @@ private void generateRefDataForEffectiveDateTesting(final Path dir) {
242266
final int finalI = i;
243267
final LocalDateTime finalEffectiveDateTime = effectiveDateTime;
244268

245-
futures.add(CompletableFuture.runAsync(() -> {
269+
runAsync(() -> {
246270
final Path refFile = makeInputFilePath(dir, finalI, finalEffectiveDateTime, refFeed);
247271
LOGGER.info("Generating file {}", refFile.toAbsolutePath().normalize().toString());
248272

@@ -263,12 +287,12 @@ private void generateRefDataForEffectiveDateTesting(final Path dir) {
263287
.consumedBy(DataGenerator.getFileOutputConsumer(refFile))
264288
.rowCount(userCount)
265289
.generate();
266-
267-
}));
290+
});
268291
effectiveDateTime = effectiveDateTime.plusDays(1);
269292
}
270293

271-
futures.add(CompletableFuture.runAsync(() -> {
294+
295+
runAsync(() -> {
272296
final Path eventsFile = makeInputFilePath(dir, 1, eventsFeed);
273297
LOGGER.info("Generating file {}", eventsFile.toAbsolutePath().normalize());
274298

@@ -290,6 +314,17 @@ private void generateRefDataForEffectiveDateTesting(final Path dir) {
290314
.rowCount(userCount)
291315
.multiThreaded()
292316
.generate();
317+
});
318+
}
319+
320+
private void runAsync(final Runnable asyncTask) {
321+
taskCounter.incrementAndGet();
322+
futures.add(CompletableFuture.runAsync(() -> {
323+
try {
324+
asyncTask.run();
325+
} finally {
326+
taskCounter.decrementAndGet();
327+
}
293328
}));
294329
}
295330

@@ -340,7 +375,7 @@ private void generateDataForCharset(final String feedName,
340375
final ByteOrderMark byteOrderMark,
341376
final String sourceContent,
342377
final int iteration) {
343-
futures.add(CompletableFuture.runAsync(() -> {
378+
runAsync(() -> {
344379
LOGGER.info("Creating feed {} in {} for charset {} and BOM {}",
345380
feedName, dir, charset, byteOrderMark);
346381

@@ -378,7 +413,7 @@ private void generateDataForCharset(final String feedName,
378413
} catch (Exception e) {
379414
throw new RuntimeException(e);
380415
}
381-
}));
416+
});
382417
}
383418

384419
private Path makeInputFilePath(final Path dir,
@@ -406,4 +441,14 @@ private void ensureAndCleanDir(final Path dir) {
406441
}
407442
}
408443

444+
445+
// --------------------------------------------------------------------------------
446+
447+
448+
private static enum DataWriterMode {
449+
XML,
450+
CSV,
451+
;
452+
}
453+
409454
}

Diff for: ‎stroom-core/src/main/java/stroom/core/receive/ReceiveDataRequestHandler.java

+2
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,9 @@ public void handle(final HttpServletRequest request, final HttpServletResponse r
142142
final StroomStatusCode stroomStatusCode = StroomStatusCode.OK;
143143
response.setStatus(stroomStatusCode.getHttpCode());
144144

145+
// TODO Should we put receiptId in content or a resp header?
145146
LOGGER.debug(() -> "Writing receipt id attribute to response: " + receiptId);
147+
// response.setHeader(StandardHeaderArguments.RECEIPT_ID, receiptId);
146148
try (final PrintWriter writer = response.getWriter()) {
147149
if (writer != null) {
148150
writer.println(receiptId);

Diff for: ‎stroom-data/stroom-data-zip/src/main/java/stroom/data/zip/StroomZipFileType.java

+35-8
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
import java.util.Locale;
77
import java.util.Map;
88
import java.util.Map.Entry;
9-
import java.util.Optional;
9+
import java.util.Objects;
1010
import java.util.Set;
11+
import java.util.function.Function;
1112
import java.util.stream.Collectors;
1213
import java.util.stream.Stream;
1314

@@ -18,6 +19,9 @@ public enum StroomZipFileType {
1819
CONTEXT(2, "ctx", "context"),
1920
DATA(3, "dat");
2021

22+
/**
23+
* Map from the canonical extension or any extensionAlias to the StroomZipFileType
24+
*/
2125
private static final Map<String, StroomZipFileType> EXTENSION_MAP = Arrays.stream(StroomZipFileType.values())
2226
.flatMap(stroomZipFileType ->
2327
Stream.concat(
@@ -26,6 +30,13 @@ public enum StroomZipFileType {
2630
.map(ext -> Map.entry(ext, stroomZipFileType)))
2731
.collect(Collectors.toMap(Entry::getKey, Entry::getValue));
2832

33+
/**
34+
* Map from the canonical extension to the StroomZipFileType, but NOT from an extensionAlias
35+
*/
36+
private static final Map<String, StroomZipFileType> CANONICAL_EXTENSION_MAP =
37+
Arrays.stream(StroomZipFileType.values())
38+
.collect(Collectors.toMap(StroomZipFileType::getExtension, Function.identity()));
39+
2940
private final int index;
3041
private final String extension;
3142
private final Set<String> extensionAliases;
@@ -70,24 +81,40 @@ public String getDotExtension() {
7081
*/
7182
public boolean hasExtension(final String fileName) {
7283
return fileName != null
73-
&& (fileName.endsWith(getDotExtension())
74-
|| getExtensionAliases().stream().anyMatch(ext -> fileName.endsWith("." + ext)));
84+
&& (fileName.endsWith(getDotExtension())
85+
|| getExtensionAliases().stream().anyMatch(ext -> fileName.endsWith("." + ext)));
7586
}
7687

7788
/**
7889
* @return True if fileName ends with the official extension for this type, but NOT an alias extension
7990
*/
8091
public boolean hasOfficialExtension(final String fileName) {
8192
return fileName != null
82-
&& fileName.endsWith(getDotExtension());
93+
&& fileName.endsWith(getDotExtension());
8394
}
8495

96+
/**
97+
* Map loosely from the canonical extension or any extensionAlias to the StroomZipFileType.
98+
* Case-insensitive.
99+
* If no match is found will return {@link StroomZipFileType#DATA}
100+
*/
85101
public static StroomZipFileType fromExtension(final String extension) {
86-
Optional<StroomZipFileType> optional = Optional.empty();
87-
if (extension != null && !extension.isEmpty()) {
88-
optional = Optional.ofNullable(EXTENSION_MAP.get(extension.toLowerCase(Locale.ROOT)));
102+
StroomZipFileType stroomZipFileType = null;
103+
if (NullSafe.isNonEmptyString(extension)) {
104+
stroomZipFileType = EXTENSION_MAP.get(extension.toLowerCase(Locale.ROOT));
89105
}
90-
return optional.orElse(StroomZipFileType.DATA);
106+
return Objects.requireNonNullElse(stroomZipFileType, StroomZipFileType.DATA);
107+
}
108+
109+
/**
110+
* Map strictly from the canonical extension to the StroomZipFileType, but NOT from an extensionAlias.
111+
* Case-sensitive.
112+
* If no match is found will return null.
113+
*/
114+
public static StroomZipFileType fromCanonicalExtension(final String canonicalExtension) {
115+
return NullSafe.isNonEmptyString(canonicalExtension)
116+
? CANONICAL_EXTENSION_MAP.get(canonicalExtension)
117+
: null;
91118
}
92119

93120
/**

0 commit comments

Comments
 (0)