Skip to content

Commit 1de0aef

Browse files
authored
HBASE-29696 Fix incorrect TableInputFormat split boundaries (#8491)
Signed-off-by: Junegunn Choi <junegunn@apache.org> Reviewed-by: Andrew Olson <930946+noslowerdna@users.noreply.github.com>
1 parent 3b378d7 commit 1de0aef

2 files changed

Lines changed: 24 additions & 0 deletions

File tree

hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,9 @@ protected List<InputSplit> createNInputSplitsUniform(InputSplit split, int n)
407407

408408
// Split Region into n chunks evenly
409409
byte[][] splitKeys = Bytes.split(startRow, endRow, true, n - 1);
410+
// Restore the original boundaries after using synthetic ones to calculate the split keys.
411+
splitKeys[0] = ts.getStartRow();
412+
splitKeys[splitKeys.length - 1] = ts.getEndRow();
410413
for (int i = 0; i < splitKeys.length - 1; i++) {
411414
// In the table input format for single table we do not need to
412415
// store the scan object in table split because it can be memory intensive and redundant

hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818
package org.apache.hadoop.hbase.mapreduce;
1919

20+
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
2021
import static org.junit.jupiter.api.Assertions.assertEquals;
2122
import static org.mockito.ArgumentMatchers.any;
2223
import static org.mockito.ArgumentMatchers.anyBoolean;
@@ -27,6 +28,7 @@
2728
import java.net.Inet6Address;
2829
import java.net.InetAddress;
2930
import java.net.UnknownHostException;
31+
import java.util.List;
3032
import java.util.Map;
3133
import java.util.TreeMap;
3234
import java.util.concurrent.ExecutorService;
@@ -52,6 +54,7 @@
5254
import org.apache.hadoop.hbase.testclassification.SmallTests;
5355
import org.apache.hadoop.hbase.util.Bytes;
5456
import org.apache.hadoop.hbase.util.Pair;
57+
import org.apache.hadoop.mapreduce.InputSplit;
5558
import org.apache.hadoop.mapreduce.JobContext;
5659
import org.junit.jupiter.api.Tag;
5760
import org.junit.jupiter.api.Test;
@@ -62,6 +65,24 @@
6265
@Tag(SmallTests.TAG)
6366
public class TestTableInputFormatBase {
6467

68+
@Test
69+
public void testCreateNInputSplitsUniformPreservesOriginalBoundaries() throws IOException {
70+
TableInputFormat inputFormat = new TableInputFormat();
71+
for (byte[] startRow : new byte[][] { HConstants.EMPTY_START_ROW, Bytes.toBytes("start") }) {
72+
TableSplit split =
73+
new TableSplit(TableName.valueOf("test"), startRow, HConstants.EMPTY_END_ROW, "localhost");
74+
75+
List<InputSplit> splits = inputFormat.createNInputSplitsUniform(split, 2);
76+
77+
assertEquals(2, splits.size());
78+
TableSplit first = (TableSplit) splits.get(0);
79+
TableSplit last = (TableSplit) splits.get(1);
80+
assertArrayEquals(startRow, first.getStartRow());
81+
assertArrayEquals(first.getEndRow(), last.getStartRow());
82+
assertArrayEquals(HConstants.EMPTY_END_ROW, last.getEndRow());
83+
}
84+
}
85+
6586
@Test
6687
public void testReuseRegionSizeCalculator() throws IOException {
6788
JobContext context = mock(JobContext.class);

0 commit comments

Comments
 (0)