Skip to content

Commit d4948fe

Browse files
committed
Size queue based on thread pool size
1 parent 3b6e8e6 commit d4948fe

File tree

1 file changed

+15
-1
lines changed

1 file changed

+15
-1
lines changed

hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public class VerifyReplication extends Configured implements Tool {
135135
public static class Verifier extends TableMapper<ImmutableBytesWritable, Put> {
136136

137137
private ThreadPoolExecutor reCompareExecutor = null;
138-
private final LinkedBlockingQueue<Future<?>> reCompareTasks = new LinkedBlockingQueue<>(1_000);
138+
private LinkedBlockingQueue<Future<?>> reCompareTasks = null;
139139

140140
public enum Counters {
141141
GOODROWS,
@@ -202,6 +202,7 @@ private ResultScanner initializeMapperOnFirstValue(Context context, Result value
202202
}
203203
int reCompareThreads = conf.getInt(NAME + ".recompareThreads", 0);
204204
reCompareExecutor = buildReCompareExecutor(reCompareThreads, context);
205+
reCompareTasks = buildReCompareTasksQueue(reCompareThreads);
205206
TableName tableName = TableName.valueOf(conf.get(NAME + ".tableName"));
206207
sourceConnection = ConnectionFactory.createConnection(conf);
207208
sourceTable = sourceConnection.getTable(tableName);
@@ -330,6 +331,15 @@ private Future<?> logFailRowAndIncreaseCounter(Context context, Counters counter
330331

331332
private void addSubmittedTaskToQueue(Future<?> task, Context context, Result value,
332333
String method) {
334+
if (reCompareTasks == null) {
335+
try {
336+
task.get();
337+
} catch (Exception e) {
338+
throw new RuntimeException(e);
339+
}
340+
return;
341+
}
342+
333343
while (true) {
334344
if (reCompareTasks.offer(task)) {
335345
break;
@@ -459,6 +469,10 @@ public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
459469
}
460470
};
461471
}
472+
473+
private static LinkedBlockingQueue<Future<?>> buildReCompareTasksQueue(int maxThreads) {
474+
return maxThreads <= 0 ? null : new LinkedBlockingQueue<>(maxThreads * 10);
475+
}
462476
}
463477

464478
private static Pair<ReplicationPeerConfig, Configuration>

0 commit comments

Comments
 (0)