Skip to content

Commit b365748

Browse files
ddupgddupg
authored
HBASE-26956 ExportSnapshot tool supports removing TTL (#4351)
Signed-off-by: Duo Zhang <[email protected]>
1 parent 7fc1674 commit b365748

File tree

4 files changed

+73
-12
lines changed

4 files changed

+73
-12
lines changed

‎hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java

+18-5
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ static final class Options {
154154
"Number of mappers to use during the copy (mapreduce.job.maps).");
155155
static final Option BANDWIDTH =
156156
new Option(null, "bandwidth", true, "Limit bandwidth to this value in MB/second.");
157+
static final Option RESET_TTL =
158+
new Option(null, "reset-ttl", false, "Do not copy TTL for the snapshot");
157159
}
158160

159161
// Export Map-Reduce Counters, to keep track of the progress
@@ -931,6 +933,7 @@ private void setPermissionParallel(final FileSystem outputFs, final short filesM
931933
private int bandwidthMB = Integer.MAX_VALUE;
932934
private int filesMode = 0;
933935
private int mappers = 0;
936+
private boolean resetTtl = false;
934937

935938
@Override
936939
protected void processOptions(CommandLine cmd) {
@@ -952,6 +955,7 @@ protected void processOptions(CommandLine cmd) {
952955
verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt());
953956
verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt());
954957
verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt());
958+
resetTtl = cmd.hasOption(Options.RESET_TTL.getLongOpt());
955959
}
956960

957961
/**
@@ -1089,11 +1093,19 @@ public int doWork() throws IOException {
10891093
}
10901094
}
10911095

1092-
// Write a new .snapshotinfo if the target name is different from the source name
1093-
if (!targetName.equals(snapshotName)) {
1094-
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils
1095-
.readSnapshotInfo(inputFs, snapshotDir).toBuilder().setName(targetName).build();
1096-
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);
1096+
// Write a new .snapshotinfo if the target name is different from the source name or we want to
1097+
// reset TTL for target snapshot.
1098+
if (!targetName.equals(snapshotName) || resetTtl) {
1099+
SnapshotDescription.Builder snapshotDescBuilder =
1100+
SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder();
1101+
if (!targetName.equals(snapshotName)) {
1102+
snapshotDescBuilder.setName(targetName);
1103+
}
1104+
if (resetTtl) {
1105+
snapshotDescBuilder.setTtl(HConstants.DEFAULT_SNAPSHOT_TTL);
1106+
}
1107+
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDescBuilder.build(),
1108+
initialOutputSnapshotDir, outputFs);
10971109
if (filesUser != null || filesGroup != null) {
10981110
outputFs.setOwner(
10991111
new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser,
@@ -1169,6 +1181,7 @@ protected void addOptions() {
11691181
addOption(Options.CHMOD);
11701182
addOption(Options.MAPPERS);
11711183
addOption(Options.BANDWIDTH);
1184+
addOption(Options.RESET_TTL);
11721185
}
11731186

11741187
public static void main(String[] args) {

‎hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java

+52-4
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,12 @@
2424

2525
import java.io.IOException;
2626
import java.util.ArrayList;
27+
import java.util.HashMap;
2728
import java.util.HashSet;
2829
import java.util.List;
30+
import java.util.Map;
2931
import java.util.Objects;
32+
import java.util.Optional;
3033
import java.util.Set;
3134
import java.util.stream.Collectors;
3235
import org.apache.hadoop.conf.Configuration;
@@ -230,6 +233,37 @@ public void testExportWithTargetName() throws Exception {
230233
testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
231234
}
232235

236+
@Test
237+
public void testExportWithResetTtl() throws Exception {
238+
String name = "testExportWithResetTtl";
239+
TableName tableName = TableName.valueOf(name);
240+
String snapshotName = "snaptb-" + name;
241+
Long ttl = 100000L;
242+
243+
try {
244+
// create Table
245+
createTable(tableName);
246+
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
247+
int tableNumFiles = admin.getRegions(tableName).size();
248+
// take a snapshot with TTL
249+
Map<String, Object> props = new HashMap<>();
250+
props.put("TTL", ttl);
251+
admin.snapshot(snapshotName, tableName, props);
252+
Optional<Long> ttlOpt =
253+
admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName))
254+
.map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
255+
assertTrue(ttlOpt.isPresent());
256+
assertEquals(ttl, ttlOpt.get());
257+
258+
testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
259+
getHdfsDestinationDir(), false, true);
260+
} catch (Exception e) {
261+
throw e;
262+
} finally {
263+
TEST_UTIL.deleteTable(tableName);
264+
}
265+
}
266+
233267
private void testExportFileSystemState(final TableName tableName, final String snapshotName,
234268
final String targetName, int filesExpected) throws Exception {
235269
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
@@ -238,8 +272,15 @@ private void testExportFileSystemState(final TableName tableName, final String s
238272

239273
protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
240274
final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
275+
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
276+
overwrite, false);
277+
}
278+
279+
protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
280+
final String targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
281+
throws Exception {
241282
testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
242-
filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite,
283+
filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
243284
getBypassRegionPredicate(), true);
244285
}
245286

@@ -249,7 +290,8 @@ protected void testExportFileSystemState(final TableName tableName, final String
249290
protected static void testExportFileSystemState(final Configuration conf,
250291
final TableName tableName, final String snapshotName, final String targetName,
251292
final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
252-
final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
293+
final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
294+
throws Exception {
253295
FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
254296
FileSystem srcFs = srcDir.getFileSystem(conf);
255297
Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
@@ -267,6 +309,9 @@ protected static void testExportFileSystemState(final Configuration conf,
267309
if (overwrite) {
268310
opts.add("--overwrite");
269311
}
312+
if (resetTtl) {
313+
opts.add("--reset-ttl");
314+
}
270315

271316
// Export Snapshot
272317
int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
@@ -295,7 +340,7 @@ protected static void testExportFileSystemState(final Configuration conf,
295340
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
296341
verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
297342
Set<String> snapshotFiles =
298-
verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, bypassregionPredicate);
343+
verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate);
299344
assertEquals(filesExpected, snapshotFiles.size());
300345
}
301346

@@ -312,7 +357,7 @@ protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
312357
*/
313358
protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
314359
final Path rootDir, final TableName tableName, final String snapshotName,
315-
final RegionPredicate bypassregionPredicate) throws IOException {
360+
final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
316361
final Path exportedSnapshot =
317362
new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
318363
final Set<String> snapshotFiles = new HashSet<>();
@@ -354,6 +399,9 @@ private void verifyNonEmptyFile(final Path path) throws IOException {
354399
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
355400
assertTrue(desc.getName().equals(snapshotName));
356401
assertTrue(desc.getTable().equals(tableName.getNameAsString()));
402+
if (resetTtl) {
403+
assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
404+
}
357405
return snapshotFiles;
358406
}
359407

‎hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ public void testExportRetry() throws Exception {
151151
conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2);
152152
conf.setInt("mapreduce.map.maxattempts", 3);
153153
TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName,
154-
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, null, true);
154+
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, null, true);
155155
}
156156

157157
/**
@@ -167,6 +167,6 @@ public void testExportFailure() throws Exception {
167167
conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4);
168168
conf.setInt("mapreduce.map.maxattempts", 3);
169169
TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName,
170-
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, null, false);
170+
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, null, false);
171171
}
172172
}

‎hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ static void testSnapshotWithRefsExportFileSystemState(FileSystem fs,
125125
TableName tableName = builder.getTableDescriptor().getTableName();
126126
TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName,
127127
snapshotName, snapshotName, snapshotFilesCount, testDir,
128-
getDestinationDir(fs, testUtil, testDir), false, null, true);
128+
getDestinationDir(fs, testUtil, testDir), false, false, null, true);
129129
}
130130

131131
static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtil hctu, Path testDir)

0 commit comments

Comments
 (0)