Skip to content

Commit eda716e

Browse files
committed
[feat] Add unit tests
1 parent e39f503 commit eda716e

File tree

5 files changed

+851
-15
lines changed

5 files changed

+851
-15
lines changed

hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadOptions.java

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.io.File;
2121
import java.lang.reflect.Field;
2222
import java.util.ArrayList;
23+
import java.util.Arrays;
2324
import java.util.List;
2425
import java.util.Set;
2526

@@ -345,8 +346,8 @@ public final class LoadOptions implements Cloneable {
345346

346347
@Parameter(names = {"--batch-failure-fallback"}, arity = 1,
347348
description = "Whether to fallback to single insert when batch insert fails. " +
348-
"Default: false")
349-
public boolean batchFailureFallback = false;
349+
"Default: true")
350+
public boolean batchFailureFallback = true;
350351

351352
public String workModeString() {
352353
if (this.incrementalMode) {
@@ -425,27 +426,28 @@ public static LoadOptions parseOptions(String[] args) {
425426
options.maxParseErrors = Constants.NO_LIMIT;
426427
options.maxInsertErrors = Constants.NO_LIMIT;
427428
}
428-
if (options.batchInsertThreads != CPUS) {
429-
adjustConnectionPoolIfDefault(options);
429+
if (Arrays.asList(args).contains("--parallel-count")) {
430+
LOG.warn("Parameter --parallel-count is deprecated, " +
431+
"please use --parser-threads instead");
430432
}
433+
adjustConnectionPoolIfDefault(options);
431434
return options;
432435
}
433436

434437
private static void adjustConnectionPoolIfDefault(LoadOptions options) {
435438
int batchThreads = options.batchInsertThreads;
436439
int maxConn = options.maxConnections;
437440
int maxConnPerRoute = options.maxConnectionsPerRoute;
441+
438442
if (maxConn == DEFAULT_MAX_CONNECTIONS && maxConn < batchThreads * 4) {
439443
options.maxConnections = batchThreads * 4;
440-
LOG.info("Auto adjusted max-conn to {} based on " +
441-
"batch-insert-threads({})",
444+
LOG.info("Auto adjusted max-conn to {} based on batch-insert-threads({})",
442445
options.maxConnections, batchThreads);
443446
}
444447

445448
if (maxConnPerRoute == DEFAULT_MAX_CONNECTIONS_PER_ROUTE && maxConnPerRoute < batchThreads * 2) {
446449
options.maxConnectionsPerRoute = batchThreads * 2;
447-
LOG.info("Auto adjusted max-conn-per-route to {} based on " +
448-
"batch-insert-threads({})",
450+
LOG.info("Auto adjusted max-conn-per-route to {} based on batch-insert-threads({})",
449451
options.maxConnectionsPerRoute, batchThreads);
450452
}
451453
}

hugegraph-loader/src/main/java/org/apache/hugegraph/loader/task/TaskManager.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,12 +138,13 @@ public void shutdown() {
138138

139139
public void submitBatch(InputStruct struct, ElementMapping mapping,
140140
List<Record> batch) {
141-
if (this.context.stopped()) {
142-
return;
143-
}
144141
long start = System.currentTimeMillis();
145142
try {
146143
this.batchSemaphore.acquire();
144+
if (this.context.stopped()) {
145+
this.batchSemaphore.release();
146+
return;
147+
}
147148
} catch (InterruptedException e) {
148149
throw new LoadException("Interrupted while waiting to submit %s " +
149150
"batch in batch mode", e, mapping.type());
@@ -162,6 +163,7 @@ public void submitBatch(InputStruct struct, ElementMapping mapping,
162163
mapping.type(), e);
163164
this.submitInSingle(struct, mapping, batch);
164165
} else {
166+
summary.metrics(struct).minusFlighting(batch.size());
165167
this.context.occurredError();
166168
this.context.stopLoading();
167169
Printer.printError("Batch insert %s failed, stop loading. Please check the logs",

hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2058,7 +2058,8 @@ public void testLoadIncrementalModeAndLoadFailure()
20582058
"-h", SERVER,
20592059
"--batch-insert-threads", "2",
20602060
"--max-parse-errors", "1",
2061-
"--test-mode", "false"
2061+
"--test-mode", "false",
2062+
"--parser-threads", "1"
20622063
));
20632064
argsList.addAll(Arrays.asList("--username", "admin", "--password", "pa"));
20642065

@@ -2259,7 +2260,8 @@ public void testReloadJsonFailureFiles() throws IOException,
22592260
"-h", SERVER,
22602261
"--check-vertex", "true",
22612262
"--batch-insert-threads", "2",
2262-
"--test-mode", "false"
2263+
"--test-mode", "false",
2264+
"--parser-threads", "1"
22632265
));
22642266
argsList.addAll(Arrays.asList("--username", "admin", "--password", "pa"));
22652267
HugeGraphLoader loader = new HugeGraphLoader(argsList.toArray(new String[0]));
@@ -2564,7 +2566,8 @@ public void testSourceOrTargetPrimaryValueNull() {
25642566
"-g", GRAPH,
25652567
"-h", SERVER,
25662568
"--batch-insert-threads", "2",
2567-
"--test-mode", "true"
2569+
"--test-mode", "true",
2570+
"--parser-threads", "1"
25682571
));
25692572

25702573
argsList.addAll(Arrays.asList("--username", "admin", "--password", "pa"));
@@ -3047,7 +3050,8 @@ public void testReadReachedMaxLines() {
30473050
"-h", SERVER,
30483051
"--max-read-lines", "4",
30493052
"--batch-insert-threads", "2",
3050-
"--test-mode", "true"
3053+
"--test-mode", "true",
3054+
"--parser-threads", "1"
30513055
};
30523056
loadWithAuth(args);
30533057

0 commit comments

Comments
 (0)