Skip to content

Commit dd32a09

Browse files
add timestamp to datalake connectors (#838)
1 parent 47e6cc9 commit dd32a09

File tree

4 files changed

+20
-8
lines changed

4 files changed

+20
-8
lines changed

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/HadoopMetadataConnector.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@
3131
import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask;
3232
import com.google.edwmigration.dumper.application.dumper.task.FormatTask;
3333
import com.google.edwmigration.dumper.application.dumper.task.Task;
34+
import com.google.edwmigration.dumper.application.dumper.utils.ArchiveNameUtil;
3435
import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description;
3536
import java.io.IOException;
37+
import java.time.Clock;
3638
import java.util.List;
3739
import java.util.stream.Stream;
3840
import javax.annotation.Nonnull;
@@ -96,6 +98,12 @@ public String getName() {
9698
return CONNECTOR_NAME;
9799
}
98100

101+
@Nonnull
102+
@Override
103+
public String getDefaultFileName(boolean isAssessment, Clock clock) {
104+
return ArchiveNameUtil.getFileNameWithTimestamp(getName() + "-metadata", clock);
105+
}
106+
99107
@Override
100108
public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArguments arguments)
101109
throws Exception {

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hdfs/HdfsExtractionConnector.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,14 @@
5050
@Description("Dumps files and directories from the HDFS.")
5151
public class HdfsExtractionConnector extends AbstractConnector implements HdfsExtractionDumpFormat {
5252

53-
public static final String CONNECTOR_NAME = "hdfs";
54-
5553
public HdfsExtractionConnector() {
56-
super(CONNECTOR_NAME);
54+
super("hdfs");
5755
}
5856

5957
@Nonnull
6058
@Override
6159
public String getDefaultFileName(boolean isAssessment, Clock clock) {
62-
return ArchiveNameUtil.getFileName(getName());
60+
return ArchiveNameUtil.getFileNameWithTimestamp(getName(), clock);
6361
}
6462

6563
@Override

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hive/HiveMetadataConnector.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import com.google.edwmigration.dumper.application.dumper.task.Task;
4343
import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
4444
import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
45+
import com.google.edwmigration.dumper.application.dumper.utils.ArchiveNameUtil;
4546
import com.google.edwmigration.dumper.ext.hive.metastore.Database;
4647
import com.google.edwmigration.dumper.ext.hive.metastore.DelegationToken;
4748
import com.google.edwmigration.dumper.ext.hive.metastore.Field;
@@ -57,6 +58,7 @@
5758
import com.google.edwmigration.dumper.plugin.lib.dumper.spi.HiveMetadataDumpFormat;
5859
import java.io.Writer;
5960
import java.nio.charset.StandardCharsets;
61+
import java.time.Clock;
6062
import java.util.ArrayList;
6163
import java.util.List;
6264
import java.util.Map;
@@ -678,10 +680,14 @@ protected String toCallDescription() {
678680
}
679681
}
680682

681-
public static final String CONNECTOR_NAME = "hiveql";
682-
683683
public HiveMetadataConnector() {
684-
super(CONNECTOR_NAME);
684+
super("hiveql");
685+
}
686+
687+
@Nonnull
688+
@Override
689+
public String getDefaultFileName(boolean isAssessment, Clock clock) {
690+
return ArchiveNameUtil.getFileNameWithTimestamp(getName() + "-metadata", clock);
685691
}
686692

687693
@Override

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ranger/RangerConnector.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public RangerConnector() {
8383
@Nonnull
8484
@Override
8585
public String getDefaultFileName(boolean isAssessment, Clock clock) {
86-
return ArchiveNameUtil.getFileName(getName());
86+
return ArchiveNameUtil.getFileNameWithTimestamp(getName(), clock);
8787
}
8888

8989
@Override

0 commit comments

Comments
 (0)